2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* SMP support for ppc.
|
|
|
|
*
|
|
|
|
* Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
|
|
|
|
* deal of code from the sparc and intel versions.
|
|
|
|
*
|
|
|
|
* Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
|
|
|
|
*
|
|
|
|
* PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
|
|
|
|
* Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#undef DEBUG
|
|
|
|
|
|
|
|
#include <linux/kernel.h>
|
2011-07-23 06:24:23 +08:00
|
|
|
#include <linux/export.h>
|
2017-02-02 02:08:20 +08:00
|
|
|
#include <linux/sched/mm.h>
|
2017-02-01 23:36:40 +08:00
|
|
|
#include <linux/sched/topology.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/smp.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/delay.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/cache.h>
|
|
|
|
#include <linux/err.h>
|
2011-12-22 06:29:42 +08:00
|
|
|
#include <linux/device.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/notifier.h>
|
2005-12-13 03:56:47 +08:00
|
|
|
#include <linux/topology.h>
|
2016-05-18 09:16:51 +08:00
|
|
|
#include <linux/profile.h>
|
2017-06-06 21:08:32 +08:00
|
|
|
#include <linux/processor.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
#include <asm/ptrace.h>
|
2011-07-27 07:09:06 +08:00
|
|
|
#include <linux/atomic.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <asm/irq.h>
|
2014-02-26 08:07:43 +08:00
|
|
|
#include <asm/hw_irq.h>
|
2014-05-23 16:15:25 +08:00
|
|
|
#include <asm/kvm_ppc.h>
|
2017-04-13 18:16:21 +08:00
|
|
|
#include <asm/dbell.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <asm/page.h>
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/prom.h>
|
|
|
|
#include <asm/smp.h>
|
|
|
|
#include <asm/time.h>
|
|
|
|
#include <asm/machdep.h>
|
2008-07-27 13:24:52 +08:00
|
|
|
#include <asm/cputhreads.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <asm/cputable.h>
|
2005-09-27 11:51:59 +08:00
|
|
|
#include <asm/mpic.h>
|
2005-11-11 18:15:21 +08:00
|
|
|
#include <asm/vdso_datapage.h>
|
2005-11-05 07:33:55 +08:00
|
|
|
#ifdef CONFIG_PPC64
|
|
|
|
#include <asm/paca.h>
|
|
|
|
#endif
|
2012-07-05 04:37:11 +08:00
|
|
|
#include <asm/vdso.h>
|
2012-03-29 01:30:02 +08:00
|
|
|
#include <asm/debug.h>
|
2014-08-20 06:55:19 +08:00
|
|
|
#include <asm/kexec.h>
|
2016-05-18 09:16:50 +08:00
|
|
|
#include <asm/asm-prototypes.h>
|
2016-07-23 17:12:40 +08:00
|
|
|
#include <asm/cpu_has_feature.h>
|
2005-11-05 07:33:55 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#ifdef DEBUG
|
2005-11-15 12:16:38 +08:00
|
|
|
#include <asm/udbg.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#define DBG(fmt...) udbg_printf(fmt)
|
|
|
|
#else
|
|
|
|
#define DBG(fmt...)
|
|
|
|
#endif
|
|
|
|
|
2011-03-08 11:40:04 +08:00
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
2011-09-20 01:44:49 +08:00
|
|
|
/* State of each CPU during hotplug phases */
|
|
|
|
static DEFINE_PER_CPU(int, cpu_state) = { 0 };
|
2011-03-08 11:40:04 +08:00
|
|
|
#endif
|
|
|
|
|
2005-11-15 12:16:38 +08:00
|
|
|
struct thread_info *secondary_ti;
|
|
|
|
|
2010-04-26 23:32:41 +08:00
|
|
|
DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
|
|
|
|
DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-10-16 16:24:05 +08:00
|
|
|
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
|
2008-07-27 13:24:53 +08:00
|
|
|
EXPORT_PER_CPU_SYMBOL(cpu_core_map);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-05 07:33:55 +08:00
|
|
|
/* SMP operations for this machine */
|
2005-04-17 06:20:36 +08:00
|
|
|
struct smp_ops_t *smp_ops;
|
|
|
|
|
2009-06-19 07:30:07 +08:00
|
|
|
/* Can't be static due to PowerMac hackery */
|
|
|
|
volatile unsigned int cpu_callin_map[NR_CPUS];
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
int smt_enabled_at_boot = 1;
|
|
|
|
|
2013-08-06 03:58:34 +08:00
|
|
|
/*
|
|
|
|
* Returns 1 if the specified cpu should be brought up during boot.
|
|
|
|
* Used to inhibit booting threads if they've been disabled or
|
|
|
|
* limited on the command line
|
|
|
|
*/
|
|
|
|
int smp_generic_cpu_bootable(unsigned int nr)
|
|
|
|
{
|
|
|
|
/* Special case - we inhibit secondary thread startup
|
|
|
|
* during boot if the user requests it.
|
|
|
|
*/
|
2017-05-17 02:42:37 +08:00
|
|
|
if (system_state < SYSTEM_RUNNING && cpu_has_feature(CPU_FTR_SMT)) {
|
2013-08-06 03:58:34 +08:00
|
|
|
if (!smt_enabled_at_boot && cpu_thread_in_core(nr) != 0)
|
|
|
|
return 0;
|
|
|
|
if (smt_enabled_at_boot
|
|
|
|
&& cpu_thread_in_core(nr) >= smt_enabled_at_boot)
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2005-11-05 07:33:55 +08:00
|
|
|
#ifdef CONFIG_PPC64
|
2012-12-22 06:04:10 +08:00
|
|
|
int smp_generic_kick_cpu(int nr)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2017-06-27 15:00:06 +08:00
|
|
|
if (nr < 0 || nr >= nr_cpu_ids)
|
2017-06-27 15:00:05 +08:00
|
|
|
return -EINVAL;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The processor is currently spinning, waiting for the
|
|
|
|
* cpu_start field to become non-zero After we set cpu_start,
|
|
|
|
* the processor will continue on to secondary_start
|
|
|
|
*/
|
2011-09-20 01:44:49 +08:00
|
|
|
if (!paca[nr].cpu_start) {
|
|
|
|
paca[nr].cpu_start = 1;
|
|
|
|
smp_mb();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
/*
|
|
|
|
* Ok it's not there, so it might be soft-unplugged, let's
|
|
|
|
* try to bring it back
|
|
|
|
*/
|
2012-07-20 20:42:34 +08:00
|
|
|
generic_set_cpu_up(nr);
|
2011-09-20 01:44:49 +08:00
|
|
|
smp_wmb();
|
|
|
|
smp_send_reschedule(nr);
|
|
|
|
#endif /* CONFIG_HOTPLUG_CPU */
|
2011-04-12 05:46:19 +08:00
|
|
|
|
|
|
|
return 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2011-09-20 01:44:49 +08:00
|
|
|
#endif /* CONFIG_PPC64 */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-11-15 04:11:49 +08:00
|
|
|
static irqreturn_t call_function_action(int irq, void *data)
|
|
|
|
{
|
|
|
|
generic_smp_call_function_interrupt();
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
static irqreturn_t reschedule_action(int irq, void *data)
|
|
|
|
{
|
2011-04-05 23:23:39 +08:00
|
|
|
scheduler_ipi();
|
2008-11-15 04:11:49 +08:00
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
2014-02-26 08:07:43 +08:00
|
|
|
static irqreturn_t tick_broadcast_ipi_action(int irq, void *data)
|
2008-11-15 04:11:49 +08:00
|
|
|
{
|
2014-02-26 08:07:43 +08:00
|
|
|
tick_broadcast_ipi_handler();
|
2008-11-15 04:11:49 +08:00
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
2016-12-20 02:30:08 +08:00
|
|
|
#ifdef CONFIG_NMI_IPI
|
|
|
|
static irqreturn_t nmi_ipi_action(int irq, void *data)
|
2008-11-15 04:11:49 +08:00
|
|
|
{
|
2016-12-20 02:30:08 +08:00
|
|
|
smp_handle_nmi_ipi(get_irq_regs());
|
2008-11-15 04:11:49 +08:00
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
2016-12-20 02:30:08 +08:00
|
|
|
#endif
|
2008-11-15 04:11:49 +08:00
|
|
|
|
|
|
|
static irq_handler_t smp_ipi_action[] = {
|
|
|
|
[PPC_MSG_CALL_FUNCTION] = call_function_action,
|
|
|
|
[PPC_MSG_RESCHEDULE] = reschedule_action,
|
2014-02-26 08:07:43 +08:00
|
|
|
[PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action,
|
2016-12-20 02:30:08 +08:00
|
|
|
#ifdef CONFIG_NMI_IPI
|
|
|
|
[PPC_MSG_NMI_IPI] = nmi_ipi_action,
|
|
|
|
#endif
|
2008-11-15 04:11:49 +08:00
|
|
|
};
|
|
|
|
|
2016-12-20 02:30:08 +08:00
|
|
|
/*
|
|
|
|
* The NMI IPI is a fallback and not truly non-maskable. It is simpler
|
|
|
|
* than going through the call function infrastructure, and strongly
|
|
|
|
* serialized, so it is more appropriate for debugging.
|
|
|
|
*/
|
2008-11-15 04:11:49 +08:00
|
|
|
const char *smp_ipi_name[] = {
|
|
|
|
[PPC_MSG_CALL_FUNCTION] = "ipi call function",
|
|
|
|
[PPC_MSG_RESCHEDULE] = "ipi reschedule",
|
2014-02-26 08:07:43 +08:00
|
|
|
[PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",
|
2016-12-20 02:30:08 +08:00
|
|
|
[PPC_MSG_NMI_IPI] = "nmi ipi",
|
2008-11-15 04:11:49 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
/* optional function to request ipi, for controllers with >= 4 ipis */
|
|
|
|
int smp_request_message_ipi(int virq, int msg)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
2016-12-20 02:30:08 +08:00
|
|
|
if (msg < 0 || msg > PPC_MSG_NMI_IPI)
|
2008-11-15 04:11:49 +08:00
|
|
|
return -EINVAL;
|
2016-12-20 02:30:08 +08:00
|
|
|
#ifndef CONFIG_NMI_IPI
|
|
|
|
if (msg == PPC_MSG_NMI_IPI)
|
2008-11-15 04:11:49 +08:00
|
|
|
return 1;
|
|
|
|
#endif
|
2016-12-20 02:30:08 +08:00
|
|
|
|
2011-10-05 10:30:50 +08:00
|
|
|
err = request_irq(virq, smp_ipi_action[msg],
|
2012-07-20 20:47:01 +08:00
|
|
|
IRQF_PERCPU | IRQF_NO_THREAD | IRQF_NO_SUSPEND,
|
2013-08-07 00:01:24 +08:00
|
|
|
smp_ipi_name[msg], NULL);
|
2008-11-15 04:11:49 +08:00
|
|
|
WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n",
|
|
|
|
virq, smp_ipi_name[msg], err);
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2011-05-11 03:29:42 +08:00
|
|
|
#ifdef CONFIG_PPC_SMP_MUXED_IPI
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
struct cpu_messages {
|
2015-12-18 04:59:03 +08:00
|
|
|
long messages; /* current messages */
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
};
|
|
|
|
static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_messages, ipi_message);
|
|
|
|
|
2015-12-18 04:59:04 +08:00
|
|
|
void smp_muxed_ipi_set_message(int cpu, int msg)
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
{
|
|
|
|
struct cpu_messages *info = &per_cpu(ipi_message, cpu);
|
2011-05-11 03:29:46 +08:00
|
|
|
char *message = (char *)&info->messages;
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
|
2012-09-05 02:33:08 +08:00
|
|
|
/*
|
|
|
|
* Order previous accesses before accesses in the IPI handler.
|
|
|
|
*/
|
|
|
|
smp_mb();
|
2011-05-11 03:29:46 +08:00
|
|
|
message[msg] = 1;
|
2015-12-18 04:59:04 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void smp_muxed_ipi_message_pass(int cpu, int msg)
|
|
|
|
{
|
|
|
|
smp_muxed_ipi_set_message(cpu, msg);
|
2017-04-13 18:16:21 +08:00
|
|
|
|
2012-09-05 02:33:08 +08:00
|
|
|
/*
|
|
|
|
* cause_ipi functions are required to include a full barrier
|
|
|
|
* before doing whatever causes the IPI.
|
|
|
|
*/
|
2017-04-13 18:16:21 +08:00
|
|
|
smp_ops->cause_ipi(cpu);
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
}
|
|
|
|
|
2013-08-07 00:01:48 +08:00
|
|
|
#ifdef __BIG_ENDIAN__
|
2015-12-18 04:59:03 +08:00
|
|
|
#define IPI_MESSAGE(A) (1uL << ((BITS_PER_LONG - 8) - 8 * (A)))
|
2013-08-07 00:01:48 +08:00
|
|
|
#else
|
2015-12-18 04:59:03 +08:00
|
|
|
#define IPI_MESSAGE(A) (1uL << (8 * (A)))
|
2013-08-07 00:01:48 +08:00
|
|
|
#endif
|
|
|
|
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
irqreturn_t smp_ipi_demux(void)
|
|
|
|
{
|
|
|
|
mb(); /* order any irq clear */
|
2011-05-11 03:29:46 +08:00
|
|
|
|
2017-04-13 18:16:22 +08:00
|
|
|
return smp_ipi_demux_relaxed();
|
|
|
|
}
|
|
|
|
|
|
|
|
/* sync-free variant. Callers should ensure synchronization */
|
|
|
|
irqreturn_t smp_ipi_demux_relaxed(void)
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
{
|
2017-04-13 18:16:21 +08:00
|
|
|
struct cpu_messages *info;
|
2015-12-18 04:59:03 +08:00
|
|
|
unsigned long all;
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
|
2017-04-13 18:16:21 +08:00
|
|
|
info = this_cpu_ptr(&ipi_message);
|
2011-05-11 03:29:46 +08:00
|
|
|
do {
|
2012-09-05 02:33:08 +08:00
|
|
|
all = xchg(&info->messages, 0);
|
2015-12-22 06:22:51 +08:00
|
|
|
#if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
|
|
|
|
/*
|
|
|
|
* Must check for PPC_MSG_RM_HOST_ACTION messages
|
|
|
|
* before PPC_MSG_CALL_FUNCTION messages because when
|
|
|
|
* a VM is destroyed, we call kick_all_cpus_sync()
|
|
|
|
* to ensure that any pending PPC_MSG_RM_HOST_ACTION
|
|
|
|
* messages have completed before we free any VCPUs.
|
|
|
|
*/
|
|
|
|
if (all & IPI_MESSAGE(PPC_MSG_RM_HOST_ACTION))
|
|
|
|
kvmppc_xics_ipi_action();
|
|
|
|
#endif
|
2013-08-07 00:01:48 +08:00
|
|
|
if (all & IPI_MESSAGE(PPC_MSG_CALL_FUNCTION))
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
generic_smp_call_function_interrupt();
|
2013-08-07 00:01:48 +08:00
|
|
|
if (all & IPI_MESSAGE(PPC_MSG_RESCHEDULE))
|
2011-05-20 13:36:52 +08:00
|
|
|
scheduler_ipi();
|
2014-02-26 08:07:43 +08:00
|
|
|
if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST))
|
|
|
|
tick_broadcast_ipi_handler();
|
2016-12-20 02:30:08 +08:00
|
|
|
#ifdef CONFIG_NMI_IPI
|
|
|
|
if (all & IPI_MESSAGE(PPC_MSG_NMI_IPI))
|
|
|
|
nmi_ipi_action(0, NULL);
|
|
|
|
#endif
|
2011-05-11 03:29:46 +08:00
|
|
|
} while (info->messages);
|
|
|
|
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
2011-05-11 03:29:42 +08:00
|
|
|
#endif /* CONFIG_PPC_SMP_MUXED_IPI */
|
powerpc: Consolidate ipi message mux and demux
Consolidate the mux and demux of ipi messages into smp.c and call
a new smp_ops callback to actually trigger the ipi.
The powerpc architecture code is optimised for having 4 distinct
ipi triggers, which are mapped to 4 distinct messages (ipi many, ipi
single, scheduler ipi, and enter debugger). However, several interrupt
controllers only provide a single software triggered interrupt that
can be delivered to each cpu. To resolve this limitation, each smp_ops
implementation created a per-cpu variable that is manipulated with atomic
bitops. Since these lines will be contended they are optimialy marked as
shared_aligned and take a full cache line for each cpu. Distro kernels
may have 2 or 3 of these in their config, each taking per-cpu space
even though at most one will be in use.
This consolidation removes smp_message_recv and replaces the single call
actions cases with direct calls from the common message recognition loop.
The complicated debugger ipi case with its muxed crash handling code is
moved to debug_ipi_action which is now called from the demux code (instead
of the multi-message action calling smp_message_recv).
I put a call to reschedule_action to increase the likelyhood of correctly
merging the anticipated scheduler_ipi() hook coming from the scheduler
tree; that single required call can be inlined later.
The actual message decode is a copy of the old pseries xics code with its
memory barriers and cache line spacing, augmented with a per-cpu unsigned
long based on the book-e doorbell code. The optional data is set via a
callback from the implementation and is passed to the new cause-ipi hook
along with the logical cpu number. While currently only the doorbell
implemntation uses this data it should be almost zero cost to retrieve and
pass it -- it adds a single register load for the argument from the same
cache line to which we just completed a store and the register is dead
on return from the call. I extended the data element from unsigned int
to unsigned long in case some other code wanted to associate a pointer.
The doorbell check_self is replaced by a call to smp_muxed_ipi_resend,
conditioned on the CPU_DBELL feature. The ifdef guard could be relaxed
to CONFIG_SMP but I left it with BOOKE for now.
Also, the doorbell interrupt vector for book-e was not calling irq_enter
and irq_exit, which throws off cpu accounting and causes code to not
realize it is running in interrupt context. Add the missing calls.
Signed-off-by: Milton Miller <miltonm@bga.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2011-05-11 03:29:39 +08:00
|
|
|
|
2011-05-26 07:34:12 +08:00
|
|
|
static inline void do_message_pass(int cpu, int msg)
|
|
|
|
{
|
|
|
|
if (smp_ops->message_pass)
|
|
|
|
smp_ops->message_pass(cpu, msg);
|
|
|
|
#ifdef CONFIG_PPC_SMP_MUXED_IPI
|
|
|
|
else
|
|
|
|
smp_muxed_ipi_message_pass(cpu, msg);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
void smp_send_reschedule(int cpu)
|
|
|
|
{
|
2006-07-04 12:09:36 +08:00
|
|
|
if (likely(smp_ops))
|
2011-05-26 07:34:12 +08:00
|
|
|
do_message_pass(cpu, PPC_MSG_RESCHEDULE);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
KVM: PPC: Add support for Book3S processors in hypervisor mode
This adds support for KVM running on 64-bit Book 3S processors,
specifically POWER7, in hypervisor mode. Using hypervisor mode means
that the guest can use the processor's supervisor mode. That means
that the guest can execute privileged instructions and access privileged
registers itself without trapping to the host. This gives excellent
performance, but does mean that KVM cannot emulate a processor
architecture other than the one that the hardware implements.
This code assumes that the guest is running paravirtualized using the
PAPR (Power Architecture Platform Requirements) interface, which is the
interface that IBM's PowerVM hypervisor uses. That means that existing
Linux distributions that run on IBM pSeries machines will also run
under KVM without modification. In order to communicate the PAPR
hypercalls to qemu, this adds a new KVM_EXIT_PAPR_HCALL exit code
to include/linux/kvm.h.
Currently the choice between book3s_hv support and book3s_pr support
(i.e. the existing code, which runs the guest in user mode) has to be
made at kernel configuration time, so a given kernel binary can only
do one or the other.
This new book3s_hv code doesn't support MMIO emulation at present.
Since we are running paravirtualized guests, this isn't a serious
restriction.
With the guest running in supervisor mode, most exceptions go straight
to the guest. We will never get data or instruction storage or segment
interrupts, alignment interrupts, decrementer interrupts, program
interrupts, single-step interrupts, etc., coming to the hypervisor from
the guest. Therefore this introduces a new KVMTEST_NONHV macro for the
exception entry path so that we don't have to do the KVM test on entry
to those exception handlers.
We do however get hypervisor decrementer, hypervisor data storage,
hypervisor instruction storage, and hypervisor emulation assist
interrupts, so we have to handle those.
In hypervisor mode, real-mode accesses can access all of RAM, not just
a limited amount. Therefore we put all the guest state in the vcpu.arch
and use the shadow_vcpu in the PACA only for temporary scratch space.
We allocate the vcpu with kzalloc rather than vzalloc, and we don't use
anything in the kvmppc_vcpu_book3s struct, so we don't allocate it.
We don't have a shared page with the guest, but we still need a
kvm_vcpu_arch_shared struct to store the values of various registers,
so we include one in the vcpu_arch struct.
The POWER7 processor has a restriction that all threads in a core have
to be in the same partition. MMU-on kernel code counts as a partition
(partition 0), so we have to do a partition switch on every entry to and
exit from the guest. At present we require the host and guest to run
in single-thread mode because of this hardware restriction.
This code allocates a hashed page table for the guest and initializes
it with HPTEs for the guest's Virtual Real Memory Area (VRMA). We
require that the guest memory is allocated using 16MB huge pages, in
order to simplify the low-level memory management. This also means that
we can get away without tracking paging activity in the host for now,
since huge pages can't be paged or swapped.
This also adds a few new exports needed by the book3s_hv code.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Alexander Graf <agraf@suse.de>
2011-06-29 08:21:34 +08:00
|
|
|
EXPORT_SYMBOL_GPL(smp_send_reschedule);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-06-26 17:22:13 +08:00
|
|
|
void arch_send_call_function_single_ipi(int cpu)
|
|
|
|
{
|
2014-02-26 08:07:29 +08:00
|
|
|
do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
|
2008-06-26 17:22:13 +08:00
|
|
|
}
|
|
|
|
|
2009-09-24 23:34:45 +08:00
|
|
|
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
|
2008-06-26 17:22:13 +08:00
|
|
|
{
|
|
|
|
unsigned int cpu;
|
|
|
|
|
2009-09-24 23:34:45 +08:00
|
|
|
for_each_cpu(cpu, mask)
|
2011-05-26 07:34:12 +08:00
|
|
|
do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
|
2008-06-26 17:22:13 +08:00
|
|
|
}
|
|
|
|
|
2016-12-20 02:30:08 +08:00
|
|
|
#ifdef CONFIG_NMI_IPI
|
|
|
|
|
|
|
|
/*
|
|
|
|
* "NMI IPI" system.
|
|
|
|
*
|
|
|
|
* NMI IPIs may not be recoverable, so should not be used as ongoing part of
|
|
|
|
* a running system. They can be used for crash, debug, halt/reboot, etc.
|
|
|
|
*
|
|
|
|
* NMI IPIs are globally single threaded. No more than one in progress at
|
|
|
|
* any time.
|
|
|
|
*
|
|
|
|
* The IPI call waits with interrupts disabled until all targets enter the
|
|
|
|
* NMI handler, then the call returns.
|
|
|
|
*
|
|
|
|
* No new NMI can be initiated until targets exit the handler.
|
|
|
|
*
|
|
|
|
* The IPI call may time out without all targets entering the NMI handler.
|
|
|
|
* In that case, there is some logic to recover (and ignore subsequent
|
|
|
|
* NMI interrupts that may eventually be raised), but the platform interrupt
|
|
|
|
* handler may not be able to distinguish this from other exception causes,
|
|
|
|
* which may cause a crash.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static atomic_t __nmi_ipi_lock = ATOMIC_INIT(0);
|
|
|
|
static struct cpumask nmi_ipi_pending_mask;
|
|
|
|
static int nmi_ipi_busy_count = 0;
|
|
|
|
static void (*nmi_ipi_function)(struct pt_regs *) = NULL;
|
|
|
|
|
|
|
|
static void nmi_ipi_lock_start(unsigned long *flags)
|
|
|
|
{
|
|
|
|
raw_local_irq_save(*flags);
|
|
|
|
hard_irq_disable();
|
|
|
|
while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) {
|
|
|
|
raw_local_irq_restore(*flags);
|
|
|
|
cpu_relax();
|
|
|
|
raw_local_irq_save(*flags);
|
|
|
|
hard_irq_disable();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nmi_ipi_lock(void)
|
|
|
|
{
|
|
|
|
while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1)
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nmi_ipi_unlock(void)
|
|
|
|
{
|
|
|
|
smp_mb();
|
|
|
|
WARN_ON(atomic_read(&__nmi_ipi_lock) != 1);
|
|
|
|
atomic_set(&__nmi_ipi_lock, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nmi_ipi_unlock_end(unsigned long *flags)
|
|
|
|
{
|
|
|
|
nmi_ipi_unlock();
|
|
|
|
raw_local_irq_restore(*flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Platform NMI handler calls this to ack
|
|
|
|
*/
|
|
|
|
int smp_handle_nmi_ipi(struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
void (*fn)(struct pt_regs *);
|
|
|
|
unsigned long flags;
|
|
|
|
int me = raw_smp_processor_id();
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Unexpected NMIs are possible here because the interrupt may not
|
|
|
|
* be able to distinguish NMI IPIs from other types of NMIs, or
|
|
|
|
* because the caller may have timed out.
|
|
|
|
*/
|
|
|
|
nmi_ipi_lock_start(&flags);
|
|
|
|
if (!nmi_ipi_busy_count)
|
|
|
|
goto out;
|
|
|
|
if (!cpumask_test_cpu(me, &nmi_ipi_pending_mask))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
fn = nmi_ipi_function;
|
|
|
|
if (!fn)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
|
|
|
|
nmi_ipi_busy_count++;
|
|
|
|
nmi_ipi_unlock();
|
|
|
|
|
|
|
|
ret = 1;
|
|
|
|
|
|
|
|
fn(regs);
|
|
|
|
|
|
|
|
nmi_ipi_lock();
|
|
|
|
nmi_ipi_busy_count--;
|
|
|
|
out:
|
|
|
|
nmi_ipi_unlock_end(&flags);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void do_smp_send_nmi_ipi(int cpu)
|
|
|
|
{
|
2016-12-20 02:30:09 +08:00
|
|
|
if (smp_ops->cause_nmi_ipi && smp_ops->cause_nmi_ipi(cpu))
|
|
|
|
return;
|
|
|
|
|
2016-12-20 02:30:08 +08:00
|
|
|
if (cpu >= 0) {
|
|
|
|
do_message_pass(cpu, PPC_MSG_NMI_IPI);
|
|
|
|
} else {
|
|
|
|
int c;
|
|
|
|
|
|
|
|
for_each_online_cpu(c) {
|
|
|
|
if (c == raw_smp_processor_id())
|
|
|
|
continue;
|
|
|
|
do_message_pass(c, PPC_MSG_NMI_IPI);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* - cpu is the target CPU (must not be this CPU), or NMI_IPI_ALL_OTHERS.
|
|
|
|
* - fn is the target callback function.
|
|
|
|
* - delay_us > 0 is the delay before giving up waiting for targets to
|
|
|
|
* enter the handler, == 0 specifies indefinite delay.
|
|
|
|
*/
|
|
|
|
static int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
int me = raw_smp_processor_id();
|
|
|
|
int ret = 1;
|
|
|
|
|
|
|
|
BUG_ON(cpu == me);
|
|
|
|
BUG_ON(cpu < 0 && cpu != NMI_IPI_ALL_OTHERS);
|
|
|
|
|
|
|
|
if (unlikely(!smp_ops))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* Take the nmi_ipi_busy count/lock with interrupts hard disabled */
|
|
|
|
nmi_ipi_lock_start(&flags);
|
|
|
|
while (nmi_ipi_busy_count) {
|
|
|
|
nmi_ipi_unlock_end(&flags);
|
|
|
|
cpu_relax();
|
|
|
|
nmi_ipi_lock_start(&flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
nmi_ipi_function = fn;
|
|
|
|
|
|
|
|
if (cpu < 0) {
|
|
|
|
/* ALL_OTHERS */
|
|
|
|
cpumask_copy(&nmi_ipi_pending_mask, cpu_online_mask);
|
|
|
|
cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
|
|
|
|
} else {
|
|
|
|
/* cpumask starts clear */
|
|
|
|
cpumask_set_cpu(cpu, &nmi_ipi_pending_mask);
|
|
|
|
}
|
|
|
|
nmi_ipi_busy_count++;
|
|
|
|
nmi_ipi_unlock();
|
|
|
|
|
|
|
|
do_smp_send_nmi_ipi(cpu);
|
|
|
|
|
|
|
|
while (!cpumask_empty(&nmi_ipi_pending_mask)) {
|
|
|
|
udelay(1);
|
|
|
|
if (delay_us) {
|
|
|
|
delay_us--;
|
|
|
|
if (!delay_us)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
nmi_ipi_lock();
|
|
|
|
if (!cpumask_empty(&nmi_ipi_pending_mask)) {
|
|
|
|
/* Could not gather all CPUs */
|
|
|
|
ret = 0;
|
|
|
|
cpumask_clear(&nmi_ipi_pending_mask);
|
|
|
|
}
|
|
|
|
nmi_ipi_busy_count--;
|
|
|
|
nmi_ipi_unlock_end(&flags);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_NMI_IPI */
|
|
|
|
|
2014-02-26 08:07:43 +08:00
|
|
|
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
|
|
|
|
void tick_broadcast(const struct cpumask *mask)
|
|
|
|
{
|
|
|
|
unsigned int cpu;
|
|
|
|
|
|
|
|
for_each_cpu(cpu, mask)
|
|
|
|
do_message_pass(cpu, PPC_MSG_TICK_BROADCAST);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2016-12-20 02:30:08 +08:00
|
|
|
#ifdef CONFIG_DEBUGGER
|
|
|
|
void debugger_ipi_callback(struct pt_regs *regs)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2016-12-20 02:30:08 +08:00
|
|
|
debugger_ipi(regs);
|
|
|
|
}
|
2011-05-11 03:29:06 +08:00
|
|
|
|
2016-12-20 02:30:08 +08:00
|
|
|
void smp_send_debugger_break(void)
|
|
|
|
{
|
|
|
|
smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, debugger_ipi_callback, 1000000);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2016-11-29 20:45:50 +08:00
|
|
|
#ifdef CONFIG_KEXEC_CORE
|
2005-12-04 15:39:43 +08:00
|
|
|
void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
|
|
|
|
{
|
2016-12-20 02:30:08 +08:00
|
|
|
smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, crash_ipi_callback, 1000000);
|
2005-12-04 15:39:43 +08:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
static void stop_this_cpu(void *dummy)
|
|
|
|
{
|
2009-11-25 19:48:52 +08:00
|
|
|
/* Remove this CPU */
|
|
|
|
set_cpu_online(smp_processor_id(), false);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
local_irq_disable();
|
|
|
|
while (1)
|
|
|
|
;
|
|
|
|
}
|
|
|
|
|
2007-09-18 07:43:40 +08:00
|
|
|
void smp_send_stop(void)
|
|
|
|
{
|
2008-06-06 17:18:06 +08:00
|
|
|
smp_call_function(stop_this_cpu, NULL, 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
struct thread_info *current_set[NR_CPUS];
|
|
|
|
|
2012-12-22 06:04:10 +08:00
|
|
|
static void smp_store_cpu_info(int id)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2009-10-29 21:34:14 +08:00
|
|
|
per_cpu(cpu_pvr, id) = mfspr(SPRN_PVR);
|
2011-06-29 03:54:47 +08:00
|
|
|
#ifdef CONFIG_PPC_FSL_BOOK3E
|
|
|
|
per_cpu(next_tlbcam_idx, id)
|
|
|
|
= (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void __init smp_prepare_cpus(unsigned int max_cpus)
|
|
|
|
{
|
|
|
|
unsigned int cpu;
|
|
|
|
|
|
|
|
DBG("smp_prepare_cpus\n");
|
|
|
|
|
|
|
|
/*
|
|
|
|
* setup_cpu may need to be called on the boot cpu. We havent
|
|
|
|
* spun any cpus up but lets be paranoid.
|
|
|
|
*/
|
|
|
|
BUG_ON(boot_cpuid != smp_processor_id());
|
|
|
|
|
|
|
|
/* Fixup boot cpu */
|
|
|
|
smp_store_cpu_info(boot_cpuid);
|
|
|
|
cpu_callin_map[boot_cpuid] = 1;
|
|
|
|
|
2010-04-26 23:32:41 +08:00
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu),
|
|
|
|
GFP_KERNEL, cpu_to_node(cpu));
|
|
|
|
zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu),
|
|
|
|
GFP_KERNEL, cpu_to_node(cpu));
|
powerpc: reorder per-cpu NUMA information's initialization
There is an issue currently where NUMA information is used on powerpc
(and possibly ia64) before it has been read from the device-tree, which
leads to large slab consumption with CONFIG_SLUB and memoryless nodes.
NUMA powerpc non-boot CPU's cpu_to_node/cpu_to_mem is only accurate
after start_secondary(), similar to ia64, which is invoked via
smp_init().
Commit 6ee0578b4daae ("workqueue: mark init_workqueues() as
early_initcall()") made init_workqueues() be invoked via
do_pre_smp_initcalls(), which is obviously before the secondary
processors are online.
Additionally, the following commits changed init_workqueues() to use
cpu_to_node to determine the node to use for kthread_create_on_node:
bce903809ab3f ("workqueue: add wq_numa_tbl_len and
wq_numa_possible_cpumask[]")
f3f90ad469342 ("workqueue: determine NUMA node of workers accourding to
the allowed cpumask")
Therefore, when init_workqueues() runs, it sees all CPUs as being on
Node 0. On LPARs or KVM guests where Node 0 is memoryless, this leads to
a high number of slab deactivations
(http://www.spinics.net/lists/linux-mm/msg67489.html).
Fix this by initializing the powerpc-specific CPU<->node/local memory
node mapping as early as possible, which on powerpc is
do_init_bootmem(). Currently that function initializes the mapping for
the boot CPU, but we extend it to setup the mapping for all possible
CPUs. Then, in smp_prepare_cpus(), we can correspondingly set the
per-cpu values for all possible CPUs. That ensures that before the
early_initcalls run (and really as early as possible), the per-cpu NUMA
mapping is accurate.
While testing memoryless nodes on PowerKVM guests with a fix to the
workqueue logic to use cpu_to_mem() instead of cpu_to_node(), with a
guest topology of:
available: 2 nodes (0-1)
node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
node 0 size: 0 MB
node 0 free: 0 MB
node 1 cpus: 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
node 1 size: 16336 MB
node 1 free: 15329 MB
node distances:
node 0 1
0: 10 40
1: 40 10
the slab consumption decreases from
Slab: 932416 kB
SUnreclaim: 902336 kB
to
Slab: 395264 kB
SUnreclaim: 359424 kB
And we a corresponding increase in the slab efficiency from
slab mem objs slabs
used active active
------------------------------------------------------------
kmalloc-16384 337 MB 11.28% 100.00%
task_struct 288 MB 9.93% 100.00%
to
slab mem objs slabs
used active active
------------------------------------------------------------
kmalloc-16384 37 MB 100.00% 100.00%
task_struct 31 MB 100.00% 100.00%
Powerpc didn't support memoryless nodes until recently (64bb80d87f01
"powerpc/numa: Enable CONFIG_HAVE_MEMORYLESS_NODES" and 8c272261194d
"powerpc/numa: Enable USE_PERCPU_NUMA_NODE_ID"). Those commits also
helped improve memory consumption with these kind of environments.
Signed-off-by: Nishanth Aravamudan <nacc@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2014-07-18 07:15:12 +08:00
|
|
|
/*
|
|
|
|
* numa_node_id() works after this.
|
|
|
|
*/
|
2014-08-27 17:34:00 +08:00
|
|
|
if (cpu_present(cpu)) {
|
|
|
|
set_cpu_numa_node(cpu, numa_cpu_lookup_table[cpu]);
|
|
|
|
set_cpu_numa_mem(cpu,
|
|
|
|
local_memory_node(numa_cpu_lookup_table[cpu]));
|
|
|
|
}
|
2010-04-26 23:32:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid));
|
|
|
|
cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid));
|
|
|
|
|
2013-07-22 14:40:20 +08:00
|
|
|
if (smp_ops && smp_ops->probe)
|
|
|
|
smp_ops->probe();
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2012-12-22 06:04:10 +08:00
|
|
|
void smp_prepare_boot_cpu(void)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
BUG_ON(smp_processor_id() != boot_cpuid);
|
2005-11-05 07:33:55 +08:00
|
|
|
#ifdef CONFIG_PPC64
|
2005-04-17 06:20:36 +08:00
|
|
|
paca[boot_cpuid].__current = current;
|
2005-11-05 07:33:55 +08:00
|
|
|
#endif
|
2014-05-20 02:14:23 +08:00
|
|
|
set_numa_node(numa_cpu_lookup_table[boot_cpuid]);
|
2006-01-12 17:06:01 +08:00
|
|
|
current_set[boot_cpuid] = task_thread_info(current);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
|
|
|
|
int generic_cpu_disable(void)
|
|
|
|
{
|
|
|
|
unsigned int cpu = smp_processor_id();
|
|
|
|
|
|
|
|
if (cpu == boot_cpuid)
|
|
|
|
return -EBUSY;
|
|
|
|
|
2009-09-24 23:34:48 +08:00
|
|
|
set_cpu_online(cpu, false);
|
2005-11-10 10:37:51 +08:00
|
|
|
#ifdef CONFIG_PPC64
|
2005-11-11 18:15:21 +08:00
|
|
|
vdso_data->processorCount--;
|
2005-11-10 11:26:12 +08:00
|
|
|
#endif
|
2017-04-05 15:54:49 +08:00
|
|
|
/* Update affinity of all IRQs previously aimed at this CPU */
|
|
|
|
irq_migrate_all_off_this_cpu();
|
|
|
|
|
2017-02-15 17:49:54 +08:00
|
|
|
/*
|
|
|
|
* Depending on the details of the interrupt controller, it's possible
|
|
|
|
* that one of the interrupts we just migrated away from this CPU is
|
|
|
|
* actually already pending on this CPU. If we leave it in that state
|
|
|
|
* the interrupt will never be EOI'ed, and will never fire again. So
|
|
|
|
* temporarily enable interrupts here, to allow any pending interrupt to
|
|
|
|
* be received (and EOI'ed), before we take this CPU offline.
|
|
|
|
*/
|
2017-04-05 15:54:49 +08:00
|
|
|
local_irq_enable();
|
|
|
|
mdelay(1);
|
|
|
|
local_irq_disable();
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void generic_cpu_die(unsigned int cpu)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < 100; i++) {
|
2005-05-01 23:58:47 +08:00
|
|
|
smp_rmb();
|
2015-11-20 17:14:01 +08:00
|
|
|
if (is_cpu_dead(cpu))
|
2005-04-17 06:20:36 +08:00
|
|
|
return;
|
|
|
|
msleep(100);
|
|
|
|
}
|
|
|
|
printk(KERN_ERR "CPU%d didn't die...\n", cpu);
|
|
|
|
}
|
|
|
|
|
2011-04-01 06:23:37 +08:00
|
|
|
void generic_set_cpu_dead(unsigned int cpu)
|
|
|
|
{
|
|
|
|
per_cpu(cpu_state, cpu) = CPU_DEAD;
|
|
|
|
}
|
2011-09-20 01:44:49 +08:00
|
|
|
|
2012-07-20 20:42:34 +08:00
|
|
|
/*
|
|
|
|
* The cpu_state should be set to CPU_UP_PREPARE in kick_cpu(), otherwise
|
|
|
|
* the cpu_state is always CPU_DEAD after calling generic_set_cpu_dead(),
|
|
|
|
* which makes the delay in generic_cpu_die() not happen.
|
|
|
|
*/
|
|
|
|
void generic_set_cpu_up(unsigned int cpu)
|
|
|
|
{
|
|
|
|
per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
|
|
|
|
}
|
|
|
|
|
2011-09-20 01:44:49 +08:00
|
|
|
int generic_check_cpu_restart(unsigned int cpu)
|
|
|
|
{
|
|
|
|
return per_cpu(cpu_state, cpu) == CPU_UP_PREPARE;
|
|
|
|
}
|
2012-10-15 09:15:41 +08:00
|
|
|
|
2015-11-20 17:14:01 +08:00
|
|
|
int is_cpu_dead(unsigned int cpu)
|
|
|
|
{
|
|
|
|
return per_cpu(cpu_state, cpu) == CPU_DEAD;
|
|
|
|
}
|
|
|
|
|
2014-05-23 16:15:25 +08:00
|
|
|
static bool secondaries_inhibited(void)
|
2012-10-15 09:15:41 +08:00
|
|
|
{
|
2014-05-23 16:15:25 +08:00
|
|
|
return kvm_hv_mode_active();
|
2012-10-15 09:15:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#else /* HOTPLUG_CPU */
|
|
|
|
|
|
|
|
#define secondaries_inhibited() 0
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
|
|
|
|
2012-04-20 21:05:48 +08:00
|
|
|
static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)
|
2011-03-08 11:40:04 +08:00
|
|
|
{
|
2012-04-20 21:05:48 +08:00
|
|
|
struct thread_info *ti = task_thread_info(idle);
|
2011-03-08 11:40:04 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_PPC64
|
2012-04-20 21:05:48 +08:00
|
|
|
paca[cpu].__current = idle;
|
2011-03-08 11:40:04 +08:00
|
|
|
paca[cpu].kstack = (unsigned long)ti + THREAD_SIZE - STACK_FRAME_OVERHEAD;
|
|
|
|
#endif
|
|
|
|
ti->cpu = cpu;
|
2012-04-20 21:05:48 +08:00
|
|
|
secondary_ti = current_set[cpu] = ti;
|
2011-03-08 11:40:04 +08:00
|
|
|
}
|
|
|
|
|
2013-06-25 03:30:09 +08:00
|
|
|
int __cpu_up(unsigned int cpu, struct task_struct *tidle)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2011-03-08 11:40:04 +08:00
|
|
|
int rc, c;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2012-10-15 09:15:41 +08:00
|
|
|
/*
|
|
|
|
* Don't allow secondary threads to come online if inhibited
|
|
|
|
*/
|
|
|
|
if (threads_per_core > 1 && secondaries_inhibited() &&
|
2014-05-23 16:15:28 +08:00
|
|
|
cpu_thread_in_subcore(cpu))
|
2012-10-15 09:15:41 +08:00
|
|
|
return -EBUSY;
|
|
|
|
|
2006-07-04 12:09:36 +08:00
|
|
|
if (smp_ops == NULL ||
|
|
|
|
(smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
|
2005-04-17 06:20:36 +08:00
|
|
|
return -EINVAL;
|
|
|
|
|
2012-04-20 21:05:48 +08:00
|
|
|
cpu_idle_thread_init(cpu, tidle);
|
2011-05-18 07:57:11 +08:00
|
|
|
|
2017-04-05 15:54:48 +08:00
|
|
|
/*
|
|
|
|
* The platform might need to allocate resources prior to bringing
|
|
|
|
* up the CPU
|
|
|
|
*/
|
|
|
|
if (smp_ops->prepare_cpu) {
|
|
|
|
rc = smp_ops->prepare_cpu(cpu);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Make sure callin-map entry is 0 (can be leftover a CPU
|
|
|
|
* hotplug
|
|
|
|
*/
|
|
|
|
cpu_callin_map[cpu] = 0;
|
|
|
|
|
|
|
|
/* The information for processor bringup must
|
|
|
|
* be written out to main store before we release
|
|
|
|
* the processor.
|
|
|
|
*/
|
2005-05-01 23:58:47 +08:00
|
|
|
smp_mb();
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* wake up cpus */
|
|
|
|
DBG("smp: kicking cpu %d\n", cpu);
|
2011-04-12 05:46:19 +08:00
|
|
|
rc = smp_ops->kick_cpu(cpu);
|
|
|
|
if (rc) {
|
|
|
|
pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc);
|
|
|
|
return rc;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* wait to see if the cpu made a callin (is actually up).
|
|
|
|
* use this value that I found through experimentation.
|
|
|
|
* -- Cort
|
|
|
|
*/
|
|
|
|
if (system_state < SYSTEM_RUNNING)
|
2006-06-18 06:52:44 +08:00
|
|
|
for (c = 50000; c && !cpu_callin_map[cpu]; c--)
|
2005-04-17 06:20:36 +08:00
|
|
|
udelay(100);
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
else
|
|
|
|
/*
|
|
|
|
* CPUs can take much longer to come up in the
|
|
|
|
* hotplug case. Wait five seconds.
|
|
|
|
*/
|
2009-06-24 07:26:37 +08:00
|
|
|
for (c = 5000; c && !cpu_callin_map[cpu]; c--)
|
|
|
|
msleep(1);
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
if (!cpu_callin_map[cpu]) {
|
2010-08-05 02:28:34 +08:00
|
|
|
printk(KERN_ERR "Processor %u is stuck.\n", cpu);
|
2005-04-17 06:20:36 +08:00
|
|
|
return -ENOENT;
|
|
|
|
}
|
|
|
|
|
2010-08-05 02:28:34 +08:00
|
|
|
DBG("Processor %u found.\n", cpu);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
if (smp_ops->give_timebase)
|
|
|
|
smp_ops->give_timebase();
|
|
|
|
|
2015-02-24 14:58:02 +08:00
|
|
|
/* Wait until cpu puts itself in the online & active maps */
|
2017-06-06 21:08:32 +08:00
|
|
|
spin_until_cond(cpu_online(cpu));
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-07-27 13:24:54 +08:00
|
|
|
/* Return the value of the reg property corresponding to the given
|
|
|
|
* logical cpu.
|
|
|
|
*/
|
|
|
|
int cpu_to_core_id(int cpu)
|
|
|
|
{
|
|
|
|
struct device_node *np;
|
2013-12-12 12:59:36 +08:00
|
|
|
const __be32 *reg;
|
2008-07-27 13:24:54 +08:00
|
|
|
int id = -1;
|
|
|
|
|
|
|
|
np = of_get_cpu_node(cpu, NULL);
|
|
|
|
if (!np)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
reg = of_get_property(np, "reg", NULL);
|
|
|
|
if (!reg)
|
|
|
|
goto out;
|
|
|
|
|
2013-12-12 12:59:36 +08:00
|
|
|
id = be32_to_cpup(reg);
|
2008-07-27 13:24:54 +08:00
|
|
|
out:
|
|
|
|
of_node_put(np);
|
|
|
|
return id;
|
|
|
|
}
|
2016-06-02 19:45:14 +08:00
|
|
|
EXPORT_SYMBOL_GPL(cpu_to_core_id);
|
2008-07-27 13:24:54 +08:00
|
|
|
|
2010-10-06 16:36:59 +08:00
|
|
|
/* Helper routines for cpu to core mapping */
|
|
|
|
int cpu_core_index_of_thread(int cpu)
|
|
|
|
{
|
|
|
|
return cpu >> threads_shift;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(cpu_core_index_of_thread);
|
|
|
|
|
|
|
|
int cpu_first_thread_of_core(int core)
|
|
|
|
{
|
|
|
|
return core << threads_shift;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(cpu_first_thread_of_core);
|
|
|
|
|
2013-08-12 14:29:33 +08:00
|
|
|
static void traverse_siblings_chip_id(int cpu, bool add, int chipid)
|
|
|
|
{
|
|
|
|
const struct cpumask *mask;
|
|
|
|
struct device_node *np;
|
|
|
|
int i, plen;
|
|
|
|
const __be32 *prop;
|
|
|
|
|
|
|
|
mask = add ? cpu_online_mask : cpu_present_mask;
|
|
|
|
for_each_cpu(i, mask) {
|
|
|
|
np = of_get_cpu_node(i, NULL);
|
|
|
|
if (!np)
|
|
|
|
continue;
|
|
|
|
prop = of_get_property(np, "ibm,chip-id", &plen);
|
|
|
|
if (prop && plen == sizeof(int) &&
|
|
|
|
of_read_number(prop, 1) == chipid) {
|
|
|
|
if (add) {
|
|
|
|
cpumask_set_cpu(cpu, cpu_core_mask(i));
|
|
|
|
cpumask_set_cpu(i, cpu_core_mask(cpu));
|
|
|
|
} else {
|
|
|
|
cpumask_clear_cpu(cpu, cpu_core_mask(i));
|
|
|
|
cpumask_clear_cpu(i, cpu_core_mask(cpu));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
of_node_put(np);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-04-28 13:07:23 +08:00
|
|
|
/* Must be called when no change can occur to cpu_present_mask,
|
2008-07-27 13:24:53 +08:00
|
|
|
* i.e. during cpu online or offline.
|
|
|
|
*/
|
|
|
|
static struct device_node *cpu_to_l2cache(int cpu)
|
|
|
|
{
|
|
|
|
struct device_node *np;
|
2008-12-11 04:16:07 +08:00
|
|
|
struct device_node *cache;
|
2008-07-27 13:24:53 +08:00
|
|
|
|
|
|
|
if (!cpu_present(cpu))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
np = of_get_cpu_node(cpu, NULL);
|
|
|
|
if (np == NULL)
|
|
|
|
return NULL;
|
|
|
|
|
2008-12-11 04:16:07 +08:00
|
|
|
cache = of_find_next_cache_node(np);
|
|
|
|
|
2008-07-27 13:24:53 +08:00
|
|
|
of_node_put(np);
|
|
|
|
|
2008-12-11 04:16:07 +08:00
|
|
|
return cache;
|
2008-07-27 13:24:53 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2013-08-12 14:28:47 +08:00
|
|
|
static void traverse_core_siblings(int cpu, bool add)
|
|
|
|
{
|
2013-08-12 14:29:33 +08:00
|
|
|
struct device_node *l2_cache, *np;
|
2013-08-12 14:28:47 +08:00
|
|
|
const struct cpumask *mask;
|
2013-08-12 14:29:33 +08:00
|
|
|
int i, chip, plen;
|
|
|
|
const __be32 *prop;
|
|
|
|
|
|
|
|
/* First see if we have ibm,chip-id properties in cpu nodes */
|
|
|
|
np = of_get_cpu_node(cpu, NULL);
|
|
|
|
if (np) {
|
|
|
|
chip = -1;
|
|
|
|
prop = of_get_property(np, "ibm,chip-id", &plen);
|
|
|
|
if (prop && plen == sizeof(int))
|
|
|
|
chip = of_read_number(prop, 1);
|
|
|
|
of_node_put(np);
|
|
|
|
if (chip >= 0) {
|
|
|
|
traverse_siblings_chip_id(cpu, add, chip);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2013-08-12 14:28:47 +08:00
|
|
|
|
|
|
|
l2_cache = cpu_to_l2cache(cpu);
|
|
|
|
mask = add ? cpu_online_mask : cpu_present_mask;
|
|
|
|
for_each_cpu(i, mask) {
|
2013-08-12 14:29:33 +08:00
|
|
|
np = cpu_to_l2cache(i);
|
2013-08-12 14:28:47 +08:00
|
|
|
if (!np)
|
|
|
|
continue;
|
|
|
|
if (np == l2_cache) {
|
|
|
|
if (add) {
|
|
|
|
cpumask_set_cpu(cpu, cpu_core_mask(i));
|
|
|
|
cpumask_set_cpu(i, cpu_core_mask(cpu));
|
|
|
|
} else {
|
|
|
|
cpumask_clear_cpu(cpu, cpu_core_mask(i));
|
|
|
|
cpumask_clear_cpu(i, cpu_core_mask(cpu));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
of_node_put(np);
|
|
|
|
}
|
|
|
|
of_node_put(l2_cache);
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Activate a secondary processor. */
|
2013-06-25 03:30:09 +08:00
|
|
|
void start_secondary(void *unused)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
unsigned int cpu = smp_processor_id();
|
2008-07-27 13:24:52 +08:00
|
|
|
int i, base;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2017-02-28 06:30:07 +08:00
|
|
|
mmgrab(&init_mm);
|
2005-04-17 06:20:36 +08:00
|
|
|
current->active_mm = &init_mm;
|
|
|
|
|
|
|
|
smp_store_cpu_info(cpu);
|
2005-11-05 07:33:55 +08:00
|
|
|
set_dec(tb_ticks_per_jiffy);
|
2005-11-10 07:45:30 +08:00
|
|
|
preempt_disable();
|
2014-12-29 12:47:05 +08:00
|
|
|
cpu_callin_map[cpu] = 1;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-09-09 01:38:52 +08:00
|
|
|
if (smp_ops->setup_cpu)
|
|
|
|
smp_ops->setup_cpu(cpu);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (smp_ops->take_timebase)
|
|
|
|
smp_ops->take_timebase();
|
|
|
|
|
2007-09-21 11:26:03 +08:00
|
|
|
secondary_cpu_time_init();
|
|
|
|
|
2011-03-08 11:49:33 +08:00
|
|
|
#ifdef CONFIG_PPC64
|
|
|
|
if (system_state == SYSTEM_RUNNING)
|
|
|
|
vdso_data->processorCount++;
|
2012-07-05 04:37:11 +08:00
|
|
|
|
|
|
|
vdso_getcpu_init();
|
2011-03-08 11:49:33 +08:00
|
|
|
#endif
|
2008-07-27 13:24:52 +08:00
|
|
|
/* Update sibling maps */
|
2010-10-06 16:36:59 +08:00
|
|
|
base = cpu_first_thread_sibling(cpu);
|
2008-07-27 13:24:52 +08:00
|
|
|
for (i = 0; i < threads_per_core; i++) {
|
powerpc: Set cpu sibling mask before online cpu
It seems following race is possible:
cpu0 cpux
smp_init->cpu_up->_cpu_up
__cpu_up
kick_cpu(1)
-------------------------------------------------------------------------
waiting online ...
... notify CPU_STARTING
set cpux active
set cpux online
-------------------------------------------------------------------------
finish waiting online
...
sched_init_smp
init_sched_domains(cpu_active_mask)
build_sched_domains
set cpux sibling info
-------------------------------------------------------------------------
Execution of cpu0 and cpux could be concurrent between two separator
lines.
So if the cpux sibling information was set too late (normally
impossible, but could be triggered by adding some delay in
start_secondary, after setting cpu online), build_sched_domains()
running on cpu0 might see cpux active, with an empty sibling mask, then
cause some bad address accessing like following:
[ 0.099855] Unable to handle kernel paging request for data at address 0xc00000038518078f
[ 0.099868] Faulting instruction address: 0xc0000000000b7a64
[ 0.099883] Oops: Kernel access of bad area, sig: 11 [#1]
[ 0.099895] PREEMPT SMP NR_CPUS=16 DEBUG_PAGEALLOC NUMA pSeries
[ 0.099922] Modules linked in:
[ 0.099940] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 3.10.0-rc1-00120-gb973425-dirty #16
[ 0.099956] task: c0000001fed80000 ti: c0000001fed7c000 task.ti: c0000001fed7c000
[ 0.099971] NIP: c0000000000b7a64 LR: c0000000000b7a40 CTR: c0000000000b4934
[ 0.099985] REGS: c0000001fed7f760 TRAP: 0300 Not tainted (3.10.0-rc1-00120-gb973425-dirty)
[ 0.099997] MSR: 8000000000009032 <SF,EE,ME,IR,DR,RI> CR: 24272828 XER: 20000003
[ 0.100045] SOFTE: 1
[ 0.100053] CFAR: c000000000445ee8
[ 0.100064] DAR: c00000038518078f, DSISR: 40000000
[ 0.100073]
GPR00: 0000000000000080 c0000001fed7f9e0 c000000000c84d48 0000000000000010
GPR04: 0000000000000010 0000000000000000 c0000001fc55e090 0000000000000000
GPR08: ffffffffffffffff c000000000b80b30 c000000000c962d8 00000003845ffc5f
GPR12: 0000000000000000 c00000000f33d000 c00000000000b9e4 0000000000000000
GPR16: 0000000000000000 0000000000000000 0000000000000001 0000000000000000
GPR20: c000000000ccf750 0000000000000000 c000000000c94d48 c0000001fc504000
GPR24: c0000001fc504000 c0000001fecef848 c000000000c94d48 c000000000ccf000
GPR28: c0000001fc522090 0000000000000010 c0000001fecef848 c0000001fed7fae0
[ 0.100293] NIP [c0000000000b7a64] .get_group+0x84/0xc4
[ 0.100307] LR [c0000000000b7a40] .get_group+0x60/0xc4
[ 0.100318] Call Trace:
[ 0.100332] [c0000001fed7f9e0] [c0000000000dbce4] .lock_is_held+0xa8/0xd0 (unreliable)
[ 0.100354] [c0000001fed7fa70] [c0000000000bf62c] .build_sched_domains+0x728/0xd14
[ 0.100375] [c0000001fed7fbe0] [c000000000af67bc] .sched_init_smp+0x4fc/0x654
[ 0.100394] [c0000001fed7fce0] [c000000000adce24] .kernel_init_freeable+0x17c/0x30c
[ 0.100413] [c0000001fed7fdb0] [c00000000000ba08] .kernel_init+0x24/0x12c
[ 0.100431] [c0000001fed7fe30] [c000000000009f74] .ret_from_kernel_thread+0x5c/0x68
[ 0.100445] Instruction dump:
[ 0.100456] 38800010 38a00000 4838e3f5 60000000 7c6307b4 2fbf0000 419e0040 3d220001
[ 0.100496] 78601f24 39491590 e93e0008 7d6a002a <7d69582a> f97f0000 7d4a002a e93e0010
[ 0.100559] ---[ end trace 31fd0ba7d8756001 ]---
This patch tries to move the sibling maps updating before
notify_cpu_starting() and cpu online, and a write barrier there to make
sure sibling maps are updated before active and online mask.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Reviewed-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-16 18:20:26 +08:00
|
|
|
if (cpu_is_offline(base + i) && (cpu != base + i))
|
2008-07-27 13:24:52 +08:00
|
|
|
continue;
|
2010-04-26 23:32:41 +08:00
|
|
|
cpumask_set_cpu(cpu, cpu_sibling_mask(base + i));
|
|
|
|
cpumask_set_cpu(base + i, cpu_sibling_mask(cpu));
|
2008-07-27 13:24:53 +08:00
|
|
|
|
|
|
|
/* cpu_core_map should be a superset of
|
|
|
|
* cpu_sibling_map even if we don't have cache
|
|
|
|
* information, so update the former here, too.
|
|
|
|
*/
|
2010-04-26 23:32:41 +08:00
|
|
|
cpumask_set_cpu(cpu, cpu_core_mask(base + i));
|
|
|
|
cpumask_set_cpu(base + i, cpu_core_mask(cpu));
|
2008-07-27 13:24:52 +08:00
|
|
|
}
|
2013-08-12 14:28:47 +08:00
|
|
|
traverse_core_siblings(cpu, true);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2014-08-27 17:34:00 +08:00
|
|
|
set_numa_node(numa_cpu_lookup_table[cpu]);
|
|
|
|
set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));
|
|
|
|
|
powerpc: Set cpu sibling mask before online cpu
It seems following race is possible:
cpu0 cpux
smp_init->cpu_up->_cpu_up
__cpu_up
kick_cpu(1)
-------------------------------------------------------------------------
waiting online ...
... notify CPU_STARTING
set cpux active
set cpux online
-------------------------------------------------------------------------
finish waiting online
...
sched_init_smp
init_sched_domains(cpu_active_mask)
build_sched_domains
set cpux sibling info
-------------------------------------------------------------------------
Execution of cpu0 and cpux could be concurrent between two separator
lines.
So if the cpux sibling information was set too late (normally
impossible, but could be triggered by adding some delay in
start_secondary, after setting cpu online), build_sched_domains()
running on cpu0 might see cpux active, with an empty sibling mask, then
cause some bad address accessing like following:
[ 0.099855] Unable to handle kernel paging request for data at address 0xc00000038518078f
[ 0.099868] Faulting instruction address: 0xc0000000000b7a64
[ 0.099883] Oops: Kernel access of bad area, sig: 11 [#1]
[ 0.099895] PREEMPT SMP NR_CPUS=16 DEBUG_PAGEALLOC NUMA pSeries
[ 0.099922] Modules linked in:
[ 0.099940] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 3.10.0-rc1-00120-gb973425-dirty #16
[ 0.099956] task: c0000001fed80000 ti: c0000001fed7c000 task.ti: c0000001fed7c000
[ 0.099971] NIP: c0000000000b7a64 LR: c0000000000b7a40 CTR: c0000000000b4934
[ 0.099985] REGS: c0000001fed7f760 TRAP: 0300 Not tainted (3.10.0-rc1-00120-gb973425-dirty)
[ 0.099997] MSR: 8000000000009032 <SF,EE,ME,IR,DR,RI> CR: 24272828 XER: 20000003
[ 0.100045] SOFTE: 1
[ 0.100053] CFAR: c000000000445ee8
[ 0.100064] DAR: c00000038518078f, DSISR: 40000000
[ 0.100073]
GPR00: 0000000000000080 c0000001fed7f9e0 c000000000c84d48 0000000000000010
GPR04: 0000000000000010 0000000000000000 c0000001fc55e090 0000000000000000
GPR08: ffffffffffffffff c000000000b80b30 c000000000c962d8 00000003845ffc5f
GPR12: 0000000000000000 c00000000f33d000 c00000000000b9e4 0000000000000000
GPR16: 0000000000000000 0000000000000000 0000000000000001 0000000000000000
GPR20: c000000000ccf750 0000000000000000 c000000000c94d48 c0000001fc504000
GPR24: c0000001fc504000 c0000001fecef848 c000000000c94d48 c000000000ccf000
GPR28: c0000001fc522090 0000000000000010 c0000001fecef848 c0000001fed7fae0
[ 0.100293] NIP [c0000000000b7a64] .get_group+0x84/0xc4
[ 0.100307] LR [c0000000000b7a40] .get_group+0x60/0xc4
[ 0.100318] Call Trace:
[ 0.100332] [c0000001fed7f9e0] [c0000000000dbce4] .lock_is_held+0xa8/0xd0 (unreliable)
[ 0.100354] [c0000001fed7fa70] [c0000000000bf62c] .build_sched_domains+0x728/0xd14
[ 0.100375] [c0000001fed7fbe0] [c000000000af67bc] .sched_init_smp+0x4fc/0x654
[ 0.100394] [c0000001fed7fce0] [c000000000adce24] .kernel_init_freeable+0x17c/0x30c
[ 0.100413] [c0000001fed7fdb0] [c00000000000ba08] .kernel_init+0x24/0x12c
[ 0.100431] [c0000001fed7fe30] [c000000000009f74] .ret_from_kernel_thread+0x5c/0x68
[ 0.100445] Instruction dump:
[ 0.100456] 38800010 38a00000 4838e3f5 60000000 7c6307b4 2fbf0000 419e0040 3d220001
[ 0.100496] 78601f24 39491590 e93e0008 7d6a002a <7d69582a> f97f0000 7d4a002a e93e0010
[ 0.100559] ---[ end trace 31fd0ba7d8756001 ]---
This patch tries to move the sibling maps updating before
notify_cpu_starting() and cpu online, and a write barrier there to make
sure sibling maps are updated before active and online mask.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Reviewed-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-16 18:20:26 +08:00
|
|
|
smp_wmb();
|
|
|
|
notify_cpu_starting(cpu);
|
|
|
|
set_cpu_online(cpu, true);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
local_irq_enable();
|
|
|
|
|
2016-02-27 02:43:40 +08:00
|
|
|
cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
|
2011-02-10 15:45:24 +08:00
|
|
|
|
|
|
|
BUG();
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
int setup_profiling_timer(unsigned int multiplier)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-04-11 17:44:39 +08:00
|
|
|
#ifdef CONFIG_SCHED_SMT
|
|
|
|
/* cpumask of CPUs with asymetric SMT dependancy */
|
2014-06-25 09:05:29 +08:00
|
|
|
static int powerpc_smt_flags(void)
|
2014-04-11 17:44:39 +08:00
|
|
|
{
|
2014-05-28 01:50:41 +08:00
|
|
|
int flags = SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
|
2014-04-11 17:44:39 +08:00
|
|
|
|
|
|
|
if (cpu_has_feature(CPU_FTR_ASYM_SMT)) {
|
|
|
|
printk_once(KERN_INFO "Enabling Asymmetric SMT scheduling\n");
|
|
|
|
flags |= SD_ASYM_PACKING;
|
|
|
|
}
|
|
|
|
return flags;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static struct sched_domain_topology_level powerpc_topology[] = {
|
|
|
|
#ifdef CONFIG_SCHED_SMT
|
|
|
|
{ cpu_smt_mask, powerpc_smt_flags, SD_INIT_NAME(SMT) },
|
|
|
|
#endif
|
|
|
|
{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
|
|
|
|
{ NULL, },
|
|
|
|
};
|
|
|
|
|
2017-04-13 04:07:31 +08:00
|
|
|
static __init long smp_setup_cpu_workfn(void *data __always_unused)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2017-04-13 04:07:31 +08:00
|
|
|
smp_ops->setup_cpu(boot_cpuid);
|
|
|
|
return 0;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2017-04-13 04:07:31 +08:00
|
|
|
void __init smp_cpus_done(unsigned int max_cpus)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We want the setup_cpu() here to be called on the boot CPU, but
|
|
|
|
* init might run on any CPU, so make sure it's invoked on the boot
|
|
|
|
* CPU.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2009-09-09 01:38:52 +08:00
|
|
|
if (smp_ops && smp_ops->setup_cpu)
|
2017-04-13 04:07:31 +08:00
|
|
|
work_on_cpu_safe(boot_cpuid, smp_setup_cpu_workfn, NULL);
|
2005-12-13 03:56:47 +08:00
|
|
|
|
2011-03-08 10:50:37 +08:00
|
|
|
if (smp_ops && smp_ops->bringup_done)
|
|
|
|
smp_ops->bringup_done();
|
|
|
|
|
2005-12-13 03:56:47 +08:00
|
|
|
dump_numa_cpu_topology();
|
2011-03-08 10:50:37 +08:00
|
|
|
|
2014-04-11 17:44:39 +08:00
|
|
|
set_sched_topology(powerpc_topology);
|
2010-08-11 04:02:05 +08:00
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
int __cpu_disable(void)
|
|
|
|
{
|
2008-07-27 13:24:52 +08:00
|
|
|
int cpu = smp_processor_id();
|
|
|
|
int base, i;
|
|
|
|
int err;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-07-27 13:24:52 +08:00
|
|
|
if (!smp_ops->cpu_disable)
|
|
|
|
return -ENOSYS;
|
|
|
|
|
|
|
|
err = smp_ops->cpu_disable();
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
/* Update sibling maps */
|
2010-10-06 16:36:59 +08:00
|
|
|
base = cpu_first_thread_sibling(cpu);
|
powerpc, hotplug: Avoid to touch non-existent cpumasks.
We observed a kernel oops when running a PPC guest with config NR_CPUS=4
and qemu option "-smp cores=1,threads=8":
[ 30.634781] Unable to handle kernel paging request for data at
address 0xc00000014192eb17
[ 30.636173] Faulting instruction address: 0xc00000000003e5cc
[ 30.637069] Oops: Kernel access of bad area, sig: 11 [#1]
[ 30.637877] SMP NR_CPUS=4 NUMA pSeries
[ 30.638471] Modules linked in:
[ 30.638949] CPU: 3 PID: 27 Comm: migration/3 Not tainted
4.7.0-07963-g9714b26 #1
[ 30.640059] task: c00000001e29c600 task.stack: c00000001e2a8000
[ 30.640956] NIP: c00000000003e5cc LR: c00000000003e550 CTR:
0000000000000000
[ 30.642001] REGS: c00000001e2ab8e0 TRAP: 0300 Not tainted
(4.7.0-07963-g9714b26)
[ 30.643139] MSR: 8000000102803033 <SF,VEC,VSX,FP,ME,IR,DR,RI,LE,TM[E]> CR: 22004084 XER: 00000000
[ 30.644583] CFAR: c000000000009e98 DAR: c00000014192eb17 DSISR: 40000000 SOFTE: 0
GPR00: c00000000140a6b8 c00000001e2abb60 c0000000016dd300 0000000000000003
GPR04: 0000000000000000 0000000000000004 c0000000016e5920 0000000000000008
GPR08: 0000000000000004 c00000014192eb17 0000000000000000 0000000000000020
GPR12: c00000000140a6c0 c00000000ffffc00 c0000000000d3ea8 c00000001e005680
GPR16: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR20: 0000000000000000 c00000001e6b3a00 0000000000000000 0000000000000001
GPR24: c00000001ff85138 c00000001ff85130 000000001eb6f000 0000000000000001
GPR28: 0000000000000000 c0000000017014e0 0000000000000000 0000000000000018
[ 30.653882] NIP [c00000000003e5cc] __cpu_disable+0xcc/0x190
[ 30.654713] LR [c00000000003e550] __cpu_disable+0x50/0x190
[ 30.655528] Call Trace:
[ 30.655893] [c00000001e2abb60] [c00000000003e550] __cpu_disable+0x50/0x190 (unreliable)
[ 30.657280] [c00000001e2abbb0] [c0000000000aca0c] take_cpu_down+0x5c/0x100
[ 30.658365] [c00000001e2abc10] [c000000000163918] multi_cpu_stop+0x1a8/0x1e0
[ 30.659617] [c00000001e2abc60] [c000000000163cc0] cpu_stopper_thread+0xf0/0x1d0
[ 30.660737] [c00000001e2abd20] [c0000000000d8d70] smpboot_thread_fn+0x290/0x2a0
[ 30.661879] [c00000001e2abd80] [c0000000000d3fa8] kthread+0x108/0x130
[ 30.662876] [c00000001e2abe30] [c000000000009968] ret_from_kernel_thread+0x5c/0x74
[ 30.664017] Instruction dump:
[ 30.664477] 7bde1f24 38a00000 787f1f24 3b600001 39890008 7d204b78 7d05e214 7d0b07b4
[ 30.665642] 796b1f24 7d26582a 7d204a14 7d29f214 <7d4048a8> 7d4a3878 7d4049ad 40c2fff4
[ 30.666854] ---[ end trace 32643b7195717741 ]---
The reason of this is that in __cpu_disable(), when we try to set the
cpu_sibling_mask or cpu_core_mask of the sibling CPUs of the disabled
one, we don't check whether the current configuration employs those
sibling CPUs(hw threads). And if a CPU is not employed by a
configuration, the percpu structures cpu_{sibling,core}_mask are not
allocated, therefore accessing those cpumasks will result in problems as
above.
This patch fixes this problem by adding an addition check on whether the
id is no less than nr_cpu_ids in the sibling CPU iteration code.
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2016-08-16 19:50:12 +08:00
|
|
|
for (i = 0; i < threads_per_core && base + i < nr_cpu_ids; i++) {
|
2010-04-26 23:32:41 +08:00
|
|
|
cpumask_clear_cpu(cpu, cpu_sibling_mask(base + i));
|
|
|
|
cpumask_clear_cpu(base + i, cpu_sibling_mask(cpu));
|
|
|
|
cpumask_clear_cpu(cpu, cpu_core_mask(base + i));
|
|
|
|
cpumask_clear_cpu(base + i, cpu_core_mask(cpu));
|
2008-07-27 13:24:53 +08:00
|
|
|
}
|
2013-08-12 14:28:47 +08:00
|
|
|
traverse_core_siblings(cpu, false);
|
2008-07-27 13:24:52 +08:00
|
|
|
|
|
|
|
return 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void __cpu_die(unsigned int cpu)
|
|
|
|
{
|
|
|
|
if (smp_ops->cpu_die)
|
|
|
|
smp_ops->cpu_die(cpu);
|
|
|
|
}
|
2010-01-14 17:52:44 +08:00
|
|
|
|
2010-05-19 10:56:29 +08:00
|
|
|
void cpu_die(void)
|
|
|
|
{
|
|
|
|
if (ppc_md.cpu_die)
|
|
|
|
ppc_md.cpu_die();
|
2011-02-10 15:45:24 +08:00
|
|
|
|
|
|
|
/* If we return, we re-enter start_secondary */
|
|
|
|
start_secondary_resume();
|
2010-05-19 10:56:29 +08:00
|
|
|
}
|
2011-02-10 15:45:24 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|