linux/arch/x86/kernel/fpu/init.c

234 lines
5.1 KiB
C
Raw Normal View History

/*
* x86 FPU boot time init code
*/
#include <asm/fpu/internal.h>
#include <asm/tlbflush.h>
/*
* The earliest FPU detection code.
*
* Set the X86_FEATURE_FPU CPU-capability bit based on
* trying to execute an actual sequence of FPU instructions:
*/
static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
{
unsigned long cr0;
u16 fsw, fcw;
fsw = fcw = 0xffff;
cr0 = read_cr0();
cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
write_cr0(cr0);
asm volatile("fninit ; fnstsw %0 ; fnstcw %1"
: "+m" (fsw), "+m" (fcw));
if (fsw == 0 && (fcw & 0x103f) == 0x003f)
set_cpu_cap(c, X86_FEATURE_FPU);
else
clear_cpu_cap(c, X86_FEATURE_FPU);
#ifndef CONFIG_MATH_EMULATION
if (!cpu_has_fpu) {
pr_emerg("No FPU found and no math emulation present\n");
pr_emerg("Giving up\n");
for (;;)
asm volatile("hlt");
}
#endif
}
/*
* Boot time FPU feature detection code:
*/
unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
unsigned int xstate_size;
EXPORT_SYMBOL_GPL(xstate_size);
static void fpu__init_system_mxcsr(void)
{
unsigned int mask = 0;
if (cpu_has_fxsr) {
struct i387_fxsave_struct fx_tmp __aligned(32) = { };
asm volatile("fxsave %0" : "+m" (fx_tmp));
mask = fx_tmp.mxcsr_mask;
/*
* If zero then use the default features mask,
* which has all features set, except the
* denormals-are-zero feature bit:
*/
if (mask == 0)
mask = 0x0000ffbf;
}
mxcsr_feature_mask &= mask;
}
/*
* Once per bootup FPU initialization sequences that will run on most x86 CPUs:
*/
static void fpu__init_system_generic(void)
{
/*
* Set up the legacy init FPU context. (xstate init might overwrite this
* with a more modern format, if the CPU supports it.)
*/
fx_finit(&init_xstate_ctx.i387);
fpu__init_system_mxcsr();
}
static void fpu__init_system_xstate_size_legacy(void)
{
static bool on_boot_cpu = 1;
if (!on_boot_cpu)
return;
on_boot_cpu = 0;
/*
* Note that xstate_size might be overwriten later during
* fpu__init_system_xstate().
*/
if (!cpu_has_fpu) {
/*
* Disable xsave as we do not support it if i387
* emulation is enabled.
*/
setup_clear_cpu_cap(X86_FEATURE_XSAVE);
setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
xstate_size = sizeof(struct i387_soft_struct);
} else {
if (cpu_has_fxsr)
xstate_size = sizeof(struct i387_fxsave_struct);
else
xstate_size = sizeof(struct i387_fsave_struct);
}
}
/*
* Initialize the TS bit in CR0 according to the style of context-switches
* we are using:
*/
static void fpu__init_cpu_ctx_switch(void)
{
if (!cpu_has_eager_fpu)
stts();
else
clts();
}
/*
* Initialize the registers found in all CPUs, CR0 and CR4:
*/
static void fpu__init_cpu_generic(void)
{
unsigned long cr0;
unsigned long cr4_mask = 0;
if (cpu_has_fxsr)
cr4_mask |= X86_CR4_OSFXSR;
if (cpu_has_xmm)
cr4_mask |= X86_CR4_OSXMMEXCPT;
if (cr4_mask)
cr4_set_bits(cr4_mask);
cr0 = read_cr0();
cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
if (!cpu_has_fpu)
cr0 |= X86_CR0_EM;
write_cr0(cr0);
}
/*
* Enable all supported FPU features. Called when a CPU is brought online.
*/
void fpu__init_cpu(void)
{
fpu__init_cpu_generic();
fpu__init_cpu_xstate();
fpu__init_cpu_ctx_switch();
x86/fpu: Split fpu__cpu_init() into early-boot and cpu-boot parts There are two kinds of FPU initialization sequences necessary to bring FPU functionality up: once per system bootup activities, such as detection, feature initialization, etc. of attributes that are shared by all CPUs in the system - and per cpu initialization sequences run when a CPU is brought online (either during bootup or during CPU hotplug onlining), such as CR0/CR4 register setting, etc. The FPU code is mixing these roles together, with no clear distinction. Start sorting this out by splitting the main FPU detection routine (fpu__cpu_init()) into two parts: fpu__init_system() for one per system init activities, and fpu__init_cpu() for the per CPU onlining init activities. Note that xstate_init() is called from both variants for the time being, because it has a dual nature as well. We'll fix that in upcoming patches. Just do the split and call it as we used to before, don't introduce any change in initialization behavior yet, beyond duplicate (and harmless) fpu__init_cpu() and xstate_init() calls - which we'll fix in later patches. Reviewed-by: Borislav Petkov <bp@alien8.de> Cc: Andy Lutomirski <luto@amacapital.net> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Fenghua Yu <fenghua.yu@intel.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-25 10:34:48 +08:00
}
static enum { AUTO, ENABLE, DISABLE } eagerfpu = AUTO;
static int __init eager_fpu_setup(char *s)
{
if (!strcmp(s, "on"))
eagerfpu = ENABLE;
else if (!strcmp(s, "off"))
eagerfpu = DISABLE;
else if (!strcmp(s, "auto"))
eagerfpu = AUTO;
return 1;
}
__setup("eagerfpu=", eager_fpu_setup);
/*
* setup_init_fpu_buf() is __init and it is OK to call it here because
* init_xstate_ctx will be unset only once during boot.
*/
static void fpu__init_system_ctx_switch(void)
{
WARN_ON(current->thread.fpu.fpstate_active);
current_thread_info()->status = 0;
/* Auto enable eagerfpu for xsaveopt */
if (cpu_has_xsaveopt && eagerfpu != DISABLE)
eagerfpu = ENABLE;
if (xfeatures_mask & XSTATE_EAGER) {
if (eagerfpu == DISABLE) {
pr_err("x86/fpu: eagerfpu switching disabled, disabling the following xstate features: 0x%llx.\n",
xfeatures_mask & XSTATE_EAGER);
xfeatures_mask &= ~XSTATE_EAGER;
} else {
eagerfpu = ENABLE;
}
}
if (eagerfpu == ENABLE)
setup_force_cpu_cap(X86_FEATURE_EAGER_FPU);
printk_once(KERN_INFO "x86/fpu: Using '%s' FPU context switches.\n", eagerfpu == ENABLE ? "eager" : "lazy");
}
x86/fpu: Split fpu__cpu_init() into early-boot and cpu-boot parts There are two kinds of FPU initialization sequences necessary to bring FPU functionality up: once per system bootup activities, such as detection, feature initialization, etc. of attributes that are shared by all CPUs in the system - and per cpu initialization sequences run when a CPU is brought online (either during bootup or during CPU hotplug onlining), such as CR0/CR4 register setting, etc. The FPU code is mixing these roles together, with no clear distinction. Start sorting this out by splitting the main FPU detection routine (fpu__cpu_init()) into two parts: fpu__init_system() for one per system init activities, and fpu__init_cpu() for the per CPU onlining init activities. Note that xstate_init() is called from both variants for the time being, because it has a dual nature as well. We'll fix that in upcoming patches. Just do the split and call it as we used to before, don't introduce any change in initialization behavior yet, beyond duplicate (and harmless) fpu__init_cpu() and xstate_init() calls - which we'll fix in later patches. Reviewed-by: Borislav Petkov <bp@alien8.de> Cc: Andy Lutomirski <luto@amacapital.net> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Fenghua Yu <fenghua.yu@intel.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-25 10:34:48 +08:00
/*
* Called on the boot CPU once per system bootup, to set up the initial FPU state that
* is later cloned into all processes.
*/
void fpu__init_system(struct cpuinfo_x86 *c)
x86/fpu: Split fpu__cpu_init() into early-boot and cpu-boot parts There are two kinds of FPU initialization sequences necessary to bring FPU functionality up: once per system bootup activities, such as detection, feature initialization, etc. of attributes that are shared by all CPUs in the system - and per cpu initialization sequences run when a CPU is brought online (either during bootup or during CPU hotplug onlining), such as CR0/CR4 register setting, etc. The FPU code is mixing these roles together, with no clear distinction. Start sorting this out by splitting the main FPU detection routine (fpu__cpu_init()) into two parts: fpu__init_system() for one per system init activities, and fpu__init_cpu() for the per CPU onlining init activities. Note that xstate_init() is called from both variants for the time being, because it has a dual nature as well. We'll fix that in upcoming patches. Just do the split and call it as we used to before, don't introduce any change in initialization behavior yet, beyond duplicate (and harmless) fpu__init_cpu() and xstate_init() calls - which we'll fix in later patches. Reviewed-by: Borislav Petkov <bp@alien8.de> Cc: Andy Lutomirski <luto@amacapital.net> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Fenghua Yu <fenghua.yu@intel.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-25 10:34:48 +08:00
{
fpu__init_system_early_generic(c);
x86/fpu: Split fpu__cpu_init() into early-boot and cpu-boot parts There are two kinds of FPU initialization sequences necessary to bring FPU functionality up: once per system bootup activities, such as detection, feature initialization, etc. of attributes that are shared by all CPUs in the system - and per cpu initialization sequences run when a CPU is brought online (either during bootup or during CPU hotplug onlining), such as CR0/CR4 register setting, etc. The FPU code is mixing these roles together, with no clear distinction. Start sorting this out by splitting the main FPU detection routine (fpu__cpu_init()) into two parts: fpu__init_system() for one per system init activities, and fpu__init_cpu() for the per CPU onlining init activities. Note that xstate_init() is called from both variants for the time being, because it has a dual nature as well. We'll fix that in upcoming patches. Just do the split and call it as we used to before, don't introduce any change in initialization behavior yet, beyond duplicate (and harmless) fpu__init_cpu() and xstate_init() calls - which we'll fix in later patches. Reviewed-by: Borislav Petkov <bp@alien8.de> Cc: Andy Lutomirski <luto@amacapital.net> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Fenghua Yu <fenghua.yu@intel.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-25 10:34:48 +08:00
/* The FPU has to be operational for some of the later FPU init activities: */
fpu__init_cpu();
/*
* But don't leave CR0::TS set yet, as some of the FPU setup methods depend
* on being able to execute FPU instructions that will fault on a set TS,
* such as the FXSAVE in fpu__init_system_mxcsr().
*/
clts();
fpu__init_system_generic();
fpu__init_system_xstate_size_legacy();
fpu__init_system_xstate();
fpu__init_system_ctx_switch();
}
static int __init no_387(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_FPU);
return 1;
}
__setup("no387", no_387);