blob: d5d44c452624c88e3abb5c75e68d00b55fee6019 [file] [log] [blame]
Ingo Molnar0c867532015-04-22 10:53:34 +02001/*
Ingo Molnarae026792015-04-26 15:36:46 +02002 * x86 FPU boot time init code:
Ingo Molnar0c867532015-04-22 10:53:34 +02003 */
Ingo Molnar78f7f1e2015-04-24 02:54:44 +02004#include <asm/fpu/internal.h>
Ingo Molnar0c867532015-04-22 10:53:34 +02005#include <asm/tlbflush.h>
yu-cheng yu4f81cba2016-01-06 14:24:51 -08006#include <asm/setup.h>
7#include <asm/cmdline.h>
Ingo Molnar0c867532015-04-22 10:53:34 +02008
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +02009#include <linux/sched.h>
Ingo Molnar29930022017-02-08 18:51:36 +010010#include <linux/sched/task.h>
yu-cheng yu4f81cba2016-01-06 14:24:51 -080011#include <linux/init.h>
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +020012
Ingo Molnarae026792015-04-26 15:36:46 +020013/*
Ingo Molnar41e78412015-04-26 15:32:40 +020014 * Initialize the registers found in all CPUs, CR0 and CR4:
15 */
16static void fpu__init_cpu_generic(void)
17{
18 unsigned long cr0;
19 unsigned long cr4_mask = 0;
20
Borislav Petkov01f8fd72016-04-04 22:25:01 +020021 if (boot_cpu_has(X86_FEATURE_FXSR))
Ingo Molnar41e78412015-04-26 15:32:40 +020022 cr4_mask |= X86_CR4_OSFXSR;
Borislav Petkovdda9edf2016-04-04 22:24:57 +020023 if (boot_cpu_has(X86_FEATURE_XMM))
Ingo Molnar41e78412015-04-26 15:32:40 +020024 cr4_mask |= X86_CR4_OSXMMEXCPT;
25 if (cr4_mask)
26 cr4_set_bits(cr4_mask);
27
28 cr0 = read_cr0();
29 cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
Borislav Petkova402a8d2016-04-04 22:24:58 +020030 if (!boot_cpu_has(X86_FEATURE_FPU))
Ingo Molnar41e78412015-04-26 15:32:40 +020031 cr0 |= X86_CR0_EM;
32 write_cr0(cr0);
Ingo Molnarb1276c42015-04-29 10:58:03 +020033
34 /* Flush out any pending x87 state: */
Ingo Molnar5fc96032015-08-22 09:52:06 +020035#ifdef CONFIG_MATH_EMULATION
Borislav Petkova402a8d2016-04-04 22:24:58 +020036 if (!boot_cpu_has(X86_FEATURE_FPU))
Ingo Molnar5fc96032015-08-22 09:52:06 +020037 fpstate_init_soft(&current->thread.fpu.state.soft);
38 else
39#endif
40 asm volatile ("fninit");
Ingo Molnar41e78412015-04-26 15:32:40 +020041}
42
43/*
Ingo Molnarae026792015-04-26 15:36:46 +020044 * Enable all supported FPU features. Called when a CPU is brought online:
Ingo Molnar41e78412015-04-26 15:32:40 +020045 */
46void fpu__init_cpu(void)
47{
48 fpu__init_cpu_generic();
49 fpu__init_cpu_xstate();
Ingo Molnar41e78412015-04-26 15:32:40 +020050}
51
Andy Lutomirski37ac78b2017-01-18 11:15:41 -080052static bool fpu__probe_without_cpuid(void)
Ingo Molnar2e2f3da2015-04-26 14:40:54 +020053{
54 unsigned long cr0;
55 u16 fsw, fcw;
56
57 fsw = fcw = 0xffff;
58
59 cr0 = read_cr0();
60 cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
61 write_cr0(cr0);
62
Andy Lutomirski37ac78b2017-01-18 11:15:41 -080063 asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw));
Ingo Molnar2e2f3da2015-04-26 14:40:54 +020064
Andy Lutomirski37ac78b2017-01-18 11:15:41 -080065 pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw);
66
67 return fsw == 0 && (fcw & 0x103f) == 0x003f;
68}
69
70static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
71{
72 if (!boot_cpu_has(X86_FEATURE_CPUID) &&
73 !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) {
74 if (fpu__probe_without_cpuid())
75 setup_force_cpu_cap(X86_FEATURE_FPU);
Andy Lutomirskif3639382016-01-21 15:24:31 -080076 else
Andy Lutomirski37ac78b2017-01-18 11:15:41 -080077 setup_clear_cpu_cap(X86_FEATURE_FPU);
Andy Lutomirskif3639382016-01-21 15:24:31 -080078 }
Ingo Molnare83ab9a2015-04-26 14:43:44 +020079
80#ifndef CONFIG_MATH_EMULATION
Andy Lutomirski97290172017-01-18 11:15:42 -080081 if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) {
Ingo Molnarae026792015-04-26 15:36:46 +020082 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
Ingo Molnare83ab9a2015-04-26 14:43:44 +020083 for (;;)
84 asm volatile("hlt");
85 }
86#endif
Ingo Molnar2e2f3da2015-04-26 14:40:54 +020087}
88
89/*
Ingo Molnar4d164092015-04-22 13:44:25 +020090 * Boot time FPU feature detection code:
91 */
Ingo Molnar0c867532015-04-22 10:53:34 +020092unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
Wanpeng Lia5758132017-05-11 02:58:55 -070093EXPORT_SYMBOL_GPL(mxcsr_feature_mask);
Ingo Molnar91a8c2a2015-04-24 10:49:11 +020094
Ingo Molnar32231872015-05-04 09:52:42 +020095static void __init fpu__init_system_mxcsr(void)
Ingo Molnar0c867532015-04-22 10:53:34 +020096{
Ingo Molnar91a8c2a2015-04-24 10:49:11 +020097 unsigned int mask = 0;
Ingo Molnar0c867532015-04-22 10:53:34 +020098
Borislav Petkov01f8fd72016-04-04 22:25:01 +020099 if (boot_cpu_has(X86_FEATURE_FXSR)) {
Ingo Molnarb96fecb2015-07-04 09:58:19 +0200100 /* Static because GCC does not get 16-byte stack alignment right: */
101 static struct fxregs_state fxregs __initdata;
Ingo Molnar91a8c2a2015-04-24 10:49:11 +0200102
Ingo Molnarb96fecb2015-07-04 09:58:19 +0200103 asm volatile("fxsave %0" : "+m" (fxregs));
Ingo Molnar91a8c2a2015-04-24 10:49:11 +0200104
Ingo Molnarb96fecb2015-07-04 09:58:19 +0200105 mask = fxregs.mxcsr_mask;
Ingo Molnar91a8c2a2015-04-24 10:49:11 +0200106
107 /*
108 * If zero then use the default features mask,
109 * which has all features set, except the
110 * denormals-are-zero feature bit:
111 */
Ingo Molnar0c867532015-04-22 10:53:34 +0200112 if (mask == 0)
113 mask = 0x0000ffbf;
114 }
115 mxcsr_feature_mask &= mask;
116}
117
Ingo Molnar7218e8b2015-04-26 14:35:54 +0200118/*
119 * Once per bootup FPU initialization sequences that will run on most x86 CPUs:
120 */
Ingo Molnar32231872015-05-04 09:52:42 +0200121static void __init fpu__init_system_generic(void)
Ingo Molnar7218e8b2015-04-26 14:35:54 +0200122{
123 /*
124 * Set up the legacy init FPU context. (xstate init might overwrite this
125 * with a more modern format, if the CPU supports it.)
126 */
Borislav Petkov6e686702016-03-11 12:32:06 +0100127 fpstate_init(&init_fpstate);
Ingo Molnar7218e8b2015-04-26 14:35:54 +0200128
129 fpu__init_system_mxcsr();
130}
131
Ingo Molnarae026792015-04-26 15:36:46 +0200132/*
133 * Size of the FPU context state. All tasks in the system use the
134 * same context size, regardless of what portion they use.
135 * This is inherent to the XSAVE architecture which puts all state
136 * components into a single, continuous memory block:
137 */
Fenghua Yubf15a8c2016-05-20 10:47:06 -0700138unsigned int fpu_kernel_xstate_size;
139EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size);
Ingo Molnar41e78412015-04-26 15:32:40 +0200140
Jiri Olsa25ec02f2015-12-21 15:25:30 +0100141/* Get alignment of the TYPE. */
142#define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)
143
144/*
145 * Enforce that 'MEMBER' is the last field of 'TYPE'.
146 *
147 * Align the computed size with alignment of the TYPE,
148 * because that's how C aligns structs.
149 */
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +0200150#define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
Jiri Olsa25ec02f2015-12-21 15:25:30 +0100151 BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \
152 TYPE_ALIGN(TYPE)))
Dave Hansen0c8c0f02015-07-17 12:28:11 +0200153
154/*
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +0200155 * We append the 'struct fpu' to the task_struct:
Dave Hansen0c8c0f02015-07-17 12:28:11 +0200156 */
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +0200157static void __init fpu__init_task_struct_size(void)
Dave Hansen0c8c0f02015-07-17 12:28:11 +0200158{
159 int task_size = sizeof(struct task_struct);
160
161 /*
162 * Subtract off the static size of the register state.
163 * It potentially has a bunch of padding.
164 */
165 task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state);
166
167 /*
168 * Add back the dynamically-calculated register state
169 * size.
170 */
Fenghua Yubf15a8c2016-05-20 10:47:06 -0700171 task_size += fpu_kernel_xstate_size;
Dave Hansen0c8c0f02015-07-17 12:28:11 +0200172
173 /*
174 * We dynamically size 'struct fpu', so we require that
175 * it be at the end of 'thread_struct' and that
176 * 'thread_struct' be at the end of 'task_struct'. If
177 * you hit a compile error here, check the structure to
178 * see if something got added to the end.
179 */
180 CHECK_MEMBER_AT_END_OF(struct fpu, state);
181 CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);
182 CHECK_MEMBER_AT_END_OF(struct task_struct, thread);
183
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +0200184 arch_task_struct_size = task_size;
Dave Hansen0c8c0f02015-07-17 12:28:11 +0200185}
186
Ingo Molnar41e78412015-04-26 15:32:40 +0200187/*
Fenghua Yubf15a8c2016-05-20 10:47:06 -0700188 * Set up the user and kernel xstate sizes based on the legacy FPU context size.
Ingo Molnar41e78412015-04-26 15:32:40 +0200189 *
190 * We set this up first, and later it will be overwritten by
191 * fpu__init_system_xstate() if the CPU knows about xstates.
192 */
Ingo Molnar32231872015-05-04 09:52:42 +0200193static void __init fpu__init_system_xstate_size_legacy(void)
Ingo Molnar0c867532015-04-22 10:53:34 +0200194{
Rasmus Villemoese49a4492015-11-13 15:18:31 +0100195 static int on_boot_cpu __initdata = 1;
Ingo Molnare97131a2015-05-05 11:34:49 +0200196
197 WARN_ON_FPU(!on_boot_cpu);
198 on_boot_cpu = 0;
199
Ingo Molnar0c867532015-04-22 10:53:34 +0200200 /*
Fenghua Yubf15a8c2016-05-20 10:47:06 -0700201 * Note that xstate sizes might be overwritten later during
Ingo Molnarc42103b2015-04-25 06:52:53 +0200202 * fpu__init_system_xstate().
Ingo Molnar0c867532015-04-22 10:53:34 +0200203 */
204
Borislav Petkova402a8d2016-04-04 22:24:58 +0200205 if (!boot_cpu_has(X86_FEATURE_FPU)) {
Ingo Molnar0c867532015-04-22 10:53:34 +0200206 /*
207 * Disable xsave as we do not support it if i387
208 * emulation is enabled.
209 */
210 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
211 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
Fenghua Yubf15a8c2016-05-20 10:47:06 -0700212 fpu_kernel_xstate_size = sizeof(struct swregs_state);
Ingo Molnar6a133202015-04-25 04:29:26 +0200213 } else {
Borislav Petkov01f8fd72016-04-04 22:25:01 +0200214 if (boot_cpu_has(X86_FEATURE_FXSR))
Fenghua Yubf15a8c2016-05-20 10:47:06 -0700215 fpu_kernel_xstate_size =
216 sizeof(struct fxregs_state);
Ingo Molnar6a133202015-04-25 04:29:26 +0200217 else
Fenghua Yubf15a8c2016-05-20 10:47:06 -0700218 fpu_kernel_xstate_size =
219 sizeof(struct fregs_state);
Ingo Molnar0c867532015-04-22 10:53:34 +0200220 }
Fenghua Yua1141e02016-05-20 10:47:05 -0700221
Fenghua Yubf15a8c2016-05-20 10:47:06 -0700222 fpu_user_xstate_size = fpu_kernel_xstate_size;
Ingo Molnar0c867532015-04-22 10:53:34 +0200223}
224
Ingo Molnarae026792015-04-26 15:36:46 +0200225/*
yu-cheng yua5fe93a2016-01-06 14:24:53 -0800226 * Find supported xfeatures based on cpu features and command-line input.
227 * This must be called after fpu__init_parse_early_param() is called and
228 * xfeatures_mask is enumerated.
229 */
230u64 __init fpu__get_supported_xfeatures_mask(void)
231{
Andy Lutomirskica6938a2016-10-04 20:34:31 -0400232 return XCNTXT_MASK;
yu-cheng yua5fe93a2016-01-06 14:24:53 -0800233}
234
Andy Lutomirskica6938a2016-10-04 20:34:31 -0400235/* Legacy code to initialize eager fpu mode. */
Ingo Molnar32231872015-05-04 09:52:42 +0200236static void __init fpu__init_system_ctx_switch(void)
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200237{
Rasmus Villemoese49a4492015-11-13 15:18:31 +0100238 static bool on_boot_cpu __initdata = 1;
Ingo Molnare97131a2015-05-05 11:34:49 +0200239
240 WARN_ON_FPU(!on_boot_cpu);
241 on_boot_cpu = 0;
242
243 WARN_ON_FPU(current->thread.fpu.fpstate_active);
Ingo Molnar6f5d2652015-04-25 20:11:05 +0200244}
245
Ingo Molnare35f6f12015-04-25 04:34:48 +0200246/*
yu-cheng yu4f81cba2016-01-06 14:24:51 -0800247 * We parse fpu parameters early because fpu__init_system() is executed
248 * before parse_early_param().
249 */
250static void __init fpu__init_parse_early_param(void)
251{
yu-cheng yu4f81cba2016-01-06 14:24:51 -0800252 if (cmdline_find_option_bool(boot_command_line, "no387"))
253 setup_clear_cpu_cap(X86_FEATURE_FPU);
254
255 if (cmdline_find_option_bool(boot_command_line, "nofxsr")) {
256 setup_clear_cpu_cap(X86_FEATURE_FXSR);
257 setup_clear_cpu_cap(X86_FEATURE_FXSR_OPT);
258 setup_clear_cpu_cap(X86_FEATURE_XMM);
259 }
260
261 if (cmdline_find_option_bool(boot_command_line, "noxsave"))
262 fpu__xstate_clear_all_cpu_caps();
263
264 if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
265 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
266
267 if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
268 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
269}
270
271/*
Ingo Molnarae026792015-04-26 15:36:46 +0200272 * Called on the boot CPU once per system bootup, to set up the initial
273 * FPU state that is later cloned into all processes:
Ingo Molnare35f6f12015-04-25 04:34:48 +0200274 */
Ingo Molnar32231872015-05-04 09:52:42 +0200275void __init fpu__init_system(struct cpuinfo_x86 *c)
Ingo Molnare35f6f12015-04-25 04:34:48 +0200276{
yu-cheng yu4f81cba2016-01-06 14:24:51 -0800277 fpu__init_parse_early_param();
Ingo Molnardd863882015-04-26 15:07:18 +0200278 fpu__init_system_early_generic(c);
279
Ingo Molnarae026792015-04-26 15:36:46 +0200280 /*
281 * The FPU has to be operational for some of the
282 * later FPU init activities:
283 */
Ingo Molnare35f6f12015-04-25 04:34:48 +0200284 fpu__init_cpu();
Ingo Molnar0c867532015-04-22 10:53:34 +0200285
Ingo Molnar7218e8b2015-04-26 14:35:54 +0200286 fpu__init_system_generic();
Ingo Molnar7638b742015-04-26 15:23:37 +0200287 fpu__init_system_xstate_size_legacy();
Ingo Molnarc42103b2015-04-25 06:52:53 +0200288 fpu__init_system_xstate();
Ingo Molnar5aaeb5c2015-07-17 12:28:12 +0200289 fpu__init_task_struct_size();
Ingo Molnar997578b2015-04-26 10:35:57 +0200290
Ingo Molnar011545b2015-04-26 08:28:31 +0200291 fpu__init_system_ctx_switch();
Ingo Molnar0c867532015-04-22 10:53:34 +0200292}