2008-10-23 13:26:29 +08:00
|
|
|
#ifndef _ASM_X86_PERCPU_H
|
|
|
|
#define _ASM_X86_PERCPU_H
|
2008-01-30 20:32:53 +08:00
|
|
|
|
2009-01-13 19:41:35 +08:00
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
extern void load_pda_offset(int cpu);
|
|
|
|
#else
|
|
|
|
static inline void load_pda_offset(int cpu) { }
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
2008-01-30 20:32:53 +08:00
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
#include <linux/compiler.h>
|
|
|
|
|
|
|
|
/* Same as asm-generic/percpu.h, except that we store the per cpu offset
|
|
|
|
in the PDA. Longer term the PDA and every per cpu variable
|
|
|
|
should be just put into a single section and referenced directly
|
|
|
|
from %gs */
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
#include <asm/pda.h>
|
|
|
|
|
|
|
|
#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
|
|
|
|
#define __my_cpu_offset read_pda(data_offset)
|
|
|
|
|
|
|
|
#define per_cpu_offset(x) (__per_cpu_offset(x))
|
|
|
|
|
2007-10-11 17:20:03 +08:00
|
|
|
#endif
|
2008-01-30 20:32:53 +08:00
|
|
|
#include <asm-generic/percpu.h>
|
|
|
|
|
|
|
|
DECLARE_PER_CPU(struct x8664_pda, pda);
|
|
|
|
|
2008-07-09 06:06:42 +08:00
|
|
|
/*
|
|
|
|
* These are supposed to be implemented as a single instruction which
|
|
|
|
* operates on the per-cpu data base segment. x86-64 doesn't have
|
|
|
|
* that yet, so this is a fairly inefficient workaround for the
|
|
|
|
* meantime. The single instruction is atomic with respect to
|
|
|
|
* preemption and interrupts, so we need to explicitly disable
|
|
|
|
* interrupts here to achieve the same effect. However, because it
|
|
|
|
* can be used from within interrupt-disable/enable, we can't actually
|
|
|
|
* disable interrupts; disabling preemption is enough.
|
|
|
|
*/
|
|
|
|
#define x86_read_percpu(var) \
|
|
|
|
({ \
|
|
|
|
typeof(per_cpu_var(var)) __tmp; \
|
|
|
|
preempt_disable(); \
|
|
|
|
__tmp = __get_cpu_var(var); \
|
|
|
|
preempt_enable(); \
|
|
|
|
__tmp; \
|
|
|
|
})
|
|
|
|
|
|
|
|
#define x86_write_percpu(var, val) \
|
|
|
|
do { \
|
|
|
|
preempt_disable(); \
|
|
|
|
__get_cpu_var(var) = (val); \
|
|
|
|
preempt_enable(); \
|
|
|
|
} while(0)
|
|
|
|
|
2008-01-30 20:32:53 +08:00
|
|
|
#else /* CONFIG_X86_64 */
|
|
|
|
|
|
|
|
#ifdef __ASSEMBLY__
|
|
|
|
|
|
|
|
/*
|
|
|
|
* PER_CPU finds an address of a per-cpu variable.
|
|
|
|
*
|
|
|
|
* Args:
|
|
|
|
* var - variable name
|
|
|
|
* reg - 32bit register
|
|
|
|
*
|
|
|
|
* The resulting address is stored in the "reg" argument.
|
|
|
|
*
|
|
|
|
* Example:
|
|
|
|
* PER_CPU(cpu_gdt_descr, %ebx)
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
#define PER_CPU(var, reg) \
|
|
|
|
movl %fs:per_cpu__##this_cpu_off, reg; \
|
|
|
|
lea per_cpu__##var(reg), reg
|
|
|
|
#define PER_CPU_VAR(var) %fs:per_cpu__##var
|
|
|
|
#else /* ! SMP */
|
|
|
|
#define PER_CPU(var, reg) \
|
|
|
|
movl $per_cpu__##var, reg
|
|
|
|
#define PER_CPU_VAR(var) per_cpu__##var
|
|
|
|
#endif /* SMP */
|
|
|
|
|
|
|
|
#else /* ...!ASSEMBLY */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* PER_CPU finds an address of a per-cpu variable.
|
|
|
|
*
|
|
|
|
* Args:
|
|
|
|
* var - variable name
|
|
|
|
* cpu - 32bit register containing the current CPU number
|
|
|
|
*
|
|
|
|
* The resulting address is stored in the "cpu" argument.
|
|
|
|
*
|
|
|
|
* Example:
|
|
|
|
* PER_CPU(cpu_gdt_descr, %ebx)
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
|
|
|
#define __my_cpu_offset x86_read_percpu(this_cpu_off)
|
|
|
|
|
|
|
|
/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
|
|
|
|
#define __percpu_seg "%%fs:"
|
|
|
|
|
|
|
|
#else /* !SMP */
|
|
|
|
|
|
|
|
#define __percpu_seg ""
|
|
|
|
|
|
|
|
#endif /* SMP */
|
|
|
|
|
|
|
|
#include <asm-generic/percpu.h>
|
|
|
|
|
|
|
|
/* We can use this directly for local CPU (faster). */
|
|
|
|
DECLARE_PER_CPU(unsigned long, this_cpu_off);
|
|
|
|
|
|
|
|
/* For arch-specific code, we can use direct single-insn ops (they
|
|
|
|
* don't give an lvalue though). */
|
|
|
|
extern void __bad_percpu_size(void);
|
|
|
|
|
2008-03-23 16:03:06 +08:00
|
|
|
#define percpu_to_op(op, var, val) \
|
|
|
|
do { \
|
|
|
|
typedef typeof(var) T__; \
|
|
|
|
if (0) { \
|
|
|
|
T__ tmp__; \
|
|
|
|
tmp__ = (val); \
|
|
|
|
} \
|
|
|
|
switch (sizeof(var)) { \
|
|
|
|
case 1: \
|
|
|
|
asm(op "b %1,"__percpu_seg"%0" \
|
|
|
|
: "+m" (var) \
|
|
|
|
: "ri" ((T__)val)); \
|
|
|
|
break; \
|
|
|
|
case 2: \
|
|
|
|
asm(op "w %1,"__percpu_seg"%0" \
|
|
|
|
: "+m" (var) \
|
|
|
|
: "ri" ((T__)val)); \
|
|
|
|
break; \
|
|
|
|
case 4: \
|
|
|
|
asm(op "l %1,"__percpu_seg"%0" \
|
|
|
|
: "+m" (var) \
|
|
|
|
: "ri" ((T__)val)); \
|
|
|
|
break; \
|
|
|
|
default: __bad_percpu_size(); \
|
|
|
|
} \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
#define percpu_from_op(op, var) \
|
|
|
|
({ \
|
|
|
|
typeof(var) ret__; \
|
|
|
|
switch (sizeof(var)) { \
|
|
|
|
case 1: \
|
|
|
|
asm(op "b "__percpu_seg"%1,%0" \
|
|
|
|
: "=r" (ret__) \
|
|
|
|
: "m" (var)); \
|
|
|
|
break; \
|
|
|
|
case 2: \
|
|
|
|
asm(op "w "__percpu_seg"%1,%0" \
|
|
|
|
: "=r" (ret__) \
|
|
|
|
: "m" (var)); \
|
|
|
|
break; \
|
|
|
|
case 4: \
|
|
|
|
asm(op "l "__percpu_seg"%1,%0" \
|
|
|
|
: "=r" (ret__) \
|
|
|
|
: "m" (var)); \
|
|
|
|
break; \
|
|
|
|
default: __bad_percpu_size(); \
|
|
|
|
} \
|
|
|
|
ret__; \
|
|
|
|
})
|
2008-01-30 20:32:53 +08:00
|
|
|
|
|
|
|
#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
|
2008-03-23 16:03:06 +08:00
|
|
|
#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu__##var, val)
|
|
|
|
#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val)
|
|
|
|
#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val)
|
|
|
|
#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val)
|
2008-01-30 20:32:53 +08:00
|
|
|
#endif /* !__ASSEMBLY__ */
|
|
|
|
#endif /* !CONFIG_X86_64 */
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-13 03:21:12 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Define the "EARLY_PER_CPU" macros. These are used for some per_cpu
|
|
|
|
* variables that are initialized and accessed before there are per_cpu
|
|
|
|
* areas allocated.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
|
|
|
|
DEFINE_PER_CPU(_type, _name) = _initvalue; \
|
|
|
|
__typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \
|
|
|
|
{ [0 ... NR_CPUS-1] = _initvalue }; \
|
x86, percpu: silence section mismatch warnings related to EARLY_PER_CPU variables
Quoting Mike Travis in "x86: cleanup early per cpu variables/accesses v4"
(23ca4bba3e20c6c3cb11c1bb0ab4770b724d39ac):
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
As these variables are NULL'ed before __init sections are dropped
(in setup_per_cpu_maps), they can be safely annotated as __ref.
This change silences following section mismatch warnings:
WARNING: vmlinux.o(.data+0x46c0): Section mismatch in reference from the variable x86_cpu_to_apicid_early_ptr to the variable .init.data:x86_cpu_to_apicid_early_map
The variable x86_cpu_to_apicid_early_ptr references
the variable __initdata x86_cpu_to_apicid_early_map
If the reference is valid then annotate the
variable with __init* (see linux/init.h) or name the variable:
*driver, *_template, *_timer, *_sht, *_ops, *_probe, *_probe_one, *_console,
WARNING: vmlinux.o(.data+0x46c8): Section mismatch in reference from the variable x86_bios_cpu_apicid_early_ptr to the variable .init.data:x86_bios_cpu_apicid_early_map
The variable x86_bios_cpu_apicid_early_ptr references
the variable __initdata x86_bios_cpu_apicid_early_map
If the reference is valid then annotate the
variable with __init* (see linux/init.h) or name the variable:
*driver, *_template, *_timer, *_sht, *_ops, *_probe, *_probe_one, *_console,
WARNING: vmlinux.o(.data+0x46d0): Section mismatch in reference from the variable x86_cpu_to_node_map_early_ptr to the variable .init.data:x86_cpu_to_node_map_early_map
The variable x86_cpu_to_node_map_early_ptr references
the variable __initdata x86_cpu_to_node_map_early_map
If the reference is valid then annotate the
variable with __init* (see linux/init.h) or name the variable:
*driver, *_template, *_timer, *_sht, *_ops, *_probe, *_probe_one, *_console,
Signed-off-by: Marcin Slusarz <marcin.slusarz@gmail.com>
Cc: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-08-17 23:50:50 +08:00
|
|
|
__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-13 03:21:12 +08:00
|
|
|
|
|
|
|
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
|
|
|
|
EXPORT_PER_CPU_SYMBOL(_name)
|
|
|
|
|
|
|
|
#define DECLARE_EARLY_PER_CPU(_type, _name) \
|
|
|
|
DECLARE_PER_CPU(_type, _name); \
|
|
|
|
extern __typeof__(_type) *_name##_early_ptr; \
|
|
|
|
extern __typeof__(_type) _name##_early_map[]
|
|
|
|
|
|
|
|
#define early_per_cpu_ptr(_name) (_name##_early_ptr)
|
|
|
|
#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
|
|
|
|
#define early_per_cpu(_name, _cpu) \
|
2009-01-13 19:41:34 +08:00
|
|
|
*(early_per_cpu_ptr(_name) ? \
|
|
|
|
&early_per_cpu_ptr(_name)[_cpu] : \
|
|
|
|
&per_cpu(_name, _cpu))
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-13 03:21:12 +08:00
|
|
|
|
|
|
|
#else /* !CONFIG_SMP */
|
|
|
|
#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
|
|
|
|
DEFINE_PER_CPU(_type, _name) = _initvalue
|
|
|
|
|
|
|
|
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
|
|
|
|
EXPORT_PER_CPU_SYMBOL(_name)
|
|
|
|
|
|
|
|
#define DECLARE_EARLY_PER_CPU(_type, _name) \
|
|
|
|
DECLARE_PER_CPU(_type, _name)
|
|
|
|
|
|
|
|
#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
|
|
|
|
#define early_per_cpu_ptr(_name) NULL
|
|
|
|
/* no early_per_cpu_map() */
|
|
|
|
|
|
|
|
#endif /* !CONFIG_SMP */
|
|
|
|
|
2008-10-23 13:26:29 +08:00
|
|
|
#endif /* _ASM_X86_PERCPU_H */
|