Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 1994 Linus Torvalds |
| 3 | * |
| 4 | * Pentium III FXSR, SSE support |
| 5 | * General FPU state handling cleanups |
| 6 | * Gareth Hughes <gareth@valinux.com>, May 2000 |
| 7 | * x86-64 work by Andi Kleen 2002 |
| 8 | */ |
| 9 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 10 | #ifndef _ASM_X86_I387_H |
| 11 | #define _ASM_X86_I387_H |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 12 | |
Herbert Xu | 3b0d659 | 2009-11-03 09:11:15 -0500 | [diff] [blame] | 13 | #ifndef __ASSEMBLY__ |
| 14 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 15 | #include <linux/sched.h> |
| 16 | #include <linux/kernel_stat.h> |
| 17 | #include <linux/regset.h> |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 18 | #include <linux/hardirq.h> |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 19 | #include <linux/slab.h> |
H. Peter Anvin | 92c37fa | 2008-02-04 16:47:58 +0100 | [diff] [blame] | 20 | #include <asm/asm.h> |
H. Peter Anvin | c9775b4 | 2010-05-11 17:49:54 -0700 | [diff] [blame] | 21 | #include <asm/cpufeature.h> |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 22 | #include <asm/processor.h> |
| 23 | #include <asm/sigcontext.h> |
| 24 | #include <asm/user.h> |
| 25 | #include <asm/uaccess.h> |
Suresh Siddha | dc1e35c | 2008-07-29 10:29:19 -0700 | [diff] [blame] | 26 | #include <asm/xsave.h> |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 27 | |
Suresh Siddha | 3c1c7f1 | 2008-07-29 10:29:21 -0700 | [diff] [blame] | 28 | extern unsigned int sig_xstate_size; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 29 | extern void fpu_init(void); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 30 | extern void mxcsr_feature_mask_init(void); |
Suresh Siddha | aa283f4 | 2008-03-10 15:28:05 -0700 | [diff] [blame] | 31 | extern int init_fpu(struct task_struct *child); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 32 | extern asmlinkage void math_state_restore(void); |
Jeremy Fitzhardinge | e6e9cac | 2009-04-24 00:40:59 -0700 | [diff] [blame] | 33 | extern void __math_state_restore(void); |
Jaswinder Singh | 3645493 | 2008-07-21 22:31:57 +0530 | [diff] [blame] | 34 | extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 35 | |
| 36 | extern user_regset_active_fn fpregs_active, xfpregs_active; |
Suresh Siddha | 5b3efd5 | 2010-02-11 11:50:59 -0800 | [diff] [blame] | 37 | extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get, |
| 38 | xstateregs_get; |
| 39 | extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set, |
| 40 | xstateregs_set; |
| 41 | |
| 42 | /* |
| 43 | * xstateregs_active == fpregs_active. Please refer to the comment |
| 44 | * at the definition of fpregs_active. |
| 45 | */ |
| 46 | #define xstateregs_active fpregs_active |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 47 | |
Suresh Siddha | c37b5ef | 2008-07-29 10:29:25 -0700 | [diff] [blame] | 48 | extern struct _fpx_sw_bytes fx_sw_reserved; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 49 | #ifdef CONFIG_IA32_EMULATION |
Suresh Siddha | 3c1c7f1 | 2008-07-29 10:29:21 -0700 | [diff] [blame] | 50 | extern unsigned int sig_xstate_ia32_size; |
Suresh Siddha | c37b5ef | 2008-07-29 10:29:25 -0700 | [diff] [blame] | 51 | extern struct _fpx_sw_bytes fx_sw_reserved_ia32; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 52 | struct _fpstate_ia32; |
Suresh Siddha | ab51370 | 2008-07-29 10:29:22 -0700 | [diff] [blame] | 53 | struct _xstate_ia32; |
| 54 | extern int save_i387_xstate_ia32(void __user *buf); |
| 55 | extern int restore_i387_xstate_ia32(void __user *buf); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 56 | #endif |
| 57 | |
Brian Gerst | 8eb91a5 | 2010-09-03 21:17:16 -0400 | [diff] [blame] | 58 | #ifdef CONFIG_MATH_EMULATION |
| 59 | extern void finit_soft_fpu(struct i387_soft_struct *soft); |
| 60 | #else |
| 61 | static inline void finit_soft_fpu(struct i387_soft_struct *soft) {} |
| 62 | #endif |
| 63 | |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 64 | #define X87_FSW_ES (1 << 7) /* Exception Summary */ |
| 65 | |
Suresh Siddha | 29104e1 | 2010-07-19 16:05:49 -0700 | [diff] [blame] | 66 | static __always_inline __pure bool use_xsaveopt(void) |
| 67 | { |
Suresh Siddha | 6bad06b | 2010-07-19 16:05:52 -0700 | [diff] [blame] | 68 | return static_cpu_has(X86_FEATURE_XSAVEOPT); |
Suresh Siddha | 29104e1 | 2010-07-19 16:05:49 -0700 | [diff] [blame] | 69 | } |
| 70 | |
H. Peter Anvin | c9775b4 | 2010-05-11 17:49:54 -0700 | [diff] [blame] | 71 | static __always_inline __pure bool use_xsave(void) |
Avi Kivity | c9ad488 | 2010-05-06 11:45:45 +0300 | [diff] [blame] | 72 | { |
H. Peter Anvin | c9775b4 | 2010-05-11 17:49:54 -0700 | [diff] [blame] | 73 | return static_cpu_has(X86_FEATURE_XSAVE); |
Avi Kivity | c9ad488 | 2010-05-06 11:45:45 +0300 | [diff] [blame] | 74 | } |
| 75 | |
Brian Gerst | 58a992b | 2010-09-03 21:17:18 -0400 | [diff] [blame] | 76 | static __always_inline __pure bool use_fxsr(void) |
| 77 | { |
| 78 | return static_cpu_has(X86_FEATURE_FXSR); |
| 79 | } |
| 80 | |
Suresh Siddha | 29104e1 | 2010-07-19 16:05:49 -0700 | [diff] [blame] | 81 | extern void __sanitize_i387_state(struct task_struct *); |
| 82 | |
| 83 | static inline void sanitize_i387_state(struct task_struct *tsk) |
| 84 | { |
| 85 | if (!use_xsaveopt()) |
| 86 | return; |
| 87 | __sanitize_i387_state(tsk); |
| 88 | } |
| 89 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 90 | #ifdef CONFIG_X86_64 |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 91 | static inline int fxrstor_checking(struct i387_fxsave_struct *fx) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 92 | { |
| 93 | int err; |
| 94 | |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 95 | /* See comment in fxsave() below. */ |
H. Peter Anvin | fd35fbc | 2010-10-22 15:33:38 -0700 | [diff] [blame] | 96 | #ifdef CONFIG_AS_FXSAVEQ |
| 97 | asm volatile("1: fxrstorq %[fx]\n\t" |
| 98 | "2:\n" |
| 99 | ".section .fixup,\"ax\"\n" |
| 100 | "3: movl $-1,%[err]\n" |
| 101 | " jmp 2b\n" |
| 102 | ".previous\n" |
| 103 | _ASM_EXTABLE(1b, 3b) |
| 104 | : [err] "=r" (err) |
| 105 | : [fx] "m" (*fx), "0" (0)); |
| 106 | #else |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 107 | asm volatile("1: rex64/fxrstor (%[fx])\n\t" |
| 108 | "2:\n" |
| 109 | ".section .fixup,\"ax\"\n" |
| 110 | "3: movl $-1,%[err]\n" |
| 111 | " jmp 2b\n" |
| 112 | ".previous\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 113 | _ASM_EXTABLE(1b, 3b) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 114 | : [err] "=r" (err) |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 115 | : [fx] "R" (fx), "m" (*fx), "0" (0)); |
H. Peter Anvin | fd35fbc | 2010-10-22 15:33:38 -0700 | [diff] [blame] | 116 | #endif |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 117 | return err; |
| 118 | } |
| 119 | |
Suresh Siddha | c37b5ef | 2008-07-29 10:29:25 -0700 | [diff] [blame] | 120 | static inline int fxsave_user(struct i387_fxsave_struct __user *fx) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 121 | { |
| 122 | int err; |
| 123 | |
Suresh Siddha | 8e221b6 | 2010-06-22 16:23:37 -0700 | [diff] [blame] | 124 | /* |
| 125 | * Clear the bytes not touched by the fxsave and reserved |
| 126 | * for the SW usage. |
| 127 | */ |
| 128 | err = __clear_user(&fx->sw_reserved, |
| 129 | sizeof(struct _fpx_sw_bytes)); |
| 130 | if (unlikely(err)) |
| 131 | return -EFAULT; |
| 132 | |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 133 | /* See comment in fxsave() below. */ |
H. Peter Anvin | fd35fbc | 2010-10-22 15:33:38 -0700 | [diff] [blame] | 134 | #ifdef CONFIG_AS_FXSAVEQ |
| 135 | asm volatile("1: fxsaveq %[fx]\n\t" |
| 136 | "2:\n" |
| 137 | ".section .fixup,\"ax\"\n" |
| 138 | "3: movl $-1,%[err]\n" |
| 139 | " jmp 2b\n" |
| 140 | ".previous\n" |
| 141 | _ASM_EXTABLE(1b, 3b) |
| 142 | : [err] "=r" (err), [fx] "=m" (*fx) |
| 143 | : "0" (0)); |
| 144 | #else |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 145 | asm volatile("1: rex64/fxsave (%[fx])\n\t" |
| 146 | "2:\n" |
| 147 | ".section .fixup,\"ax\"\n" |
| 148 | "3: movl $-1,%[err]\n" |
| 149 | " jmp 2b\n" |
| 150 | ".previous\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 151 | _ASM_EXTABLE(1b, 3b) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 152 | : [err] "=r" (err), "=m" (*fx) |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 153 | : [fx] "R" (fx), "0" (0)); |
H. Peter Anvin | fd35fbc | 2010-10-22 15:33:38 -0700 | [diff] [blame] | 154 | #endif |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 155 | if (unlikely(err) && |
| 156 | __clear_user(fx, sizeof(struct i387_fxsave_struct))) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 157 | err = -EFAULT; |
| 158 | /* No need to clear here because the caller clears USED_MATH */ |
| 159 | return err; |
| 160 | } |
| 161 | |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 162 | static inline void fpu_fxsave(struct fpu *fpu) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 163 | { |
| 164 | /* Using "rex64; fxsave %0" is broken because, if the memory operand |
| 165 | uses any extended registers for addressing, a second REX prefix |
| 166 | will be generated (to the assembler, rex64 followed by semicolon |
| 167 | is a separate instruction), and hence the 64-bitness is lost. */ |
Linus Torvalds | b6f7e38 | 2010-10-21 13:34:32 -0700 | [diff] [blame] | 168 | |
H. Peter Anvin | d7acb92 | 2010-10-13 16:00:29 -0700 | [diff] [blame] | 169 | #ifdef CONFIG_AS_FXSAVEQ |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 170 | /* Using "fxsaveq %0" would be the ideal choice, but is only supported |
| 171 | starting with gas 2.16. */ |
| 172 | __asm__ __volatile__("fxsaveq %0" |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 173 | : "=m" (fpu->state->fxsave)); |
Linus Torvalds | b6f7e38 | 2010-10-21 13:34:32 -0700 | [diff] [blame] | 174 | #else |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 175 | /* Using, as a workaround, the properly prefixed form below isn't |
| 176 | accepted by any binutils version so far released, complaining that |
| 177 | the same type of prefix is used twice if an extended register is |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 178 | needed for addressing (fix submitted to mainline 2005-11-21). |
| 179 | asm volatile("rex64/fxsave %0" |
| 180 | : "=m" (fpu->state->fxsave)); |
| 181 | This, however, we can work around by forcing the compiler to select |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 182 | an addressing mode that doesn't require extended registers. */ |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 183 | asm volatile("rex64/fxsave (%[fx])" |
| 184 | : "=m" (fpu->state->fxsave) |
| 185 | : [fx] "R" (&fpu->state->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 186 | #endif |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 187 | } |
| 188 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 189 | #else /* CONFIG_X86_32 */ |
| 190 | |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 191 | /* perform fxrstor iff the processor has extended states, otherwise frstor */ |
| 192 | static inline int fxrstor_checking(struct i387_fxsave_struct *fx) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 193 | { |
| 194 | /* |
| 195 | * The "nop" is needed to make the instructions the same |
| 196 | * length. |
| 197 | */ |
| 198 | alternative_input( |
| 199 | "nop ; frstor %1", |
| 200 | "fxrstor %1", |
| 201 | X86_FEATURE_FXSR, |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 202 | "m" (*fx)); |
| 203 | |
Jiri Slaby | fcb2ac5 | 2009-04-08 13:31:58 +0200 | [diff] [blame] | 204 | return 0; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 205 | } |
| 206 | |
Brian Gerst | 58a992b | 2010-09-03 21:17:18 -0400 | [diff] [blame] | 207 | static inline void fpu_fxsave(struct fpu *fpu) |
| 208 | { |
| 209 | asm volatile("fxsave %[fx]" |
| 210 | : [fx] "=m" (fpu->state->fxsave)); |
| 211 | } |
| 212 | |
Brian Gerst | b2b57fe | 2010-09-03 21:17:19 -0400 | [diff] [blame] | 213 | #endif /* CONFIG_X86_64 */ |
| 214 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 215 | /* We need a safe address that is cheap to find and that is already |
| 216 | in L1 during context switch. The best choices are unfortunately |
| 217 | different for UP and SMP */ |
| 218 | #ifdef CONFIG_SMP |
| 219 | #define safe_address (__per_cpu_offset[0]) |
| 220 | #else |
| 221 | #define safe_address (kstat_cpu(0).cpustat.user) |
| 222 | #endif |
| 223 | |
| 224 | /* |
| 225 | * These must be called with preempt disabled |
| 226 | */ |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 227 | static inline void fpu_save_init(struct fpu *fpu) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 228 | { |
Avi Kivity | c9ad488 | 2010-05-06 11:45:45 +0300 | [diff] [blame] | 229 | if (use_xsave()) { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 230 | fpu_xsave(fpu); |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 231 | |
| 232 | /* |
| 233 | * xsave header may indicate the init state of the FP. |
| 234 | */ |
Brian Gerst | 58a992b | 2010-09-03 21:17:18 -0400 | [diff] [blame] | 235 | if (!(fpu->state->xsave.xsave_hdr.xstate_bv & XSTATE_FP)) |
| 236 | return; |
| 237 | } else if (use_fxsr()) { |
| 238 | fpu_fxsave(fpu); |
| 239 | } else { |
Hans Rosenfeld | f994d99 | 2011-04-06 18:06:43 +0200 | [diff] [blame] | 240 | asm volatile("fnsave %[fx]; fwait" |
Brian Gerst | 58a992b | 2010-09-03 21:17:18 -0400 | [diff] [blame] | 241 | : [fx] "=m" (fpu->state->fsave)); |
| 242 | return; |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 243 | } |
| 244 | |
Brian Gerst | 58a992b | 2010-09-03 21:17:18 -0400 | [diff] [blame] | 245 | if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES)) |
| 246 | asm volatile("fnclex"); |
| 247 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 248 | /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception |
| 249 | is pending. Clear the x87 state here by setting it to fixed |
| 250 | values. safe_address is a random variable that should be in L1 */ |
| 251 | alternative_input( |
Brian Gerst | b2b57fe | 2010-09-03 21:17:19 -0400 | [diff] [blame] | 252 | ASM_NOP8 ASM_NOP2, |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 253 | "emms\n\t" /* clear stack tags */ |
Brian Gerst | b2b57fe | 2010-09-03 21:17:19 -0400 | [diff] [blame] | 254 | "fildl %P[addr]", /* set F?P to defined value */ |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 255 | X86_FEATURE_FXSAVE_LEAK, |
| 256 | [addr] "m" (safe_address)); |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 257 | } |
| 258 | |
| 259 | static inline void __save_init_fpu(struct task_struct *tsk) |
| 260 | { |
| 261 | fpu_save_init(&tsk->thread.fpu); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 262 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 263 | } |
| 264 | |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 265 | static inline int fpu_fxrstor_checking(struct fpu *fpu) |
| 266 | { |
| 267 | return fxrstor_checking(&fpu->state->fxsave); |
| 268 | } |
| 269 | |
| 270 | static inline int fpu_restore_checking(struct fpu *fpu) |
| 271 | { |
| 272 | if (use_xsave()) |
| 273 | return fpu_xrstor_checking(fpu); |
| 274 | else |
| 275 | return fpu_fxrstor_checking(fpu); |
| 276 | } |
| 277 | |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 278 | static inline int restore_fpu_checking(struct task_struct *tsk) |
| 279 | { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 280 | return fpu_restore_checking(&tsk->thread.fpu); |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 281 | } |
| 282 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 283 | /* |
| 284 | * Signal frame handlers... |
| 285 | */ |
Suresh Siddha | ab51370 | 2008-07-29 10:29:22 -0700 | [diff] [blame] | 286 | extern int save_i387_xstate(void __user *buf); |
| 287 | extern int restore_i387_xstate(void __user *buf); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 288 | |
| 289 | static inline void __unlazy_fpu(struct task_struct *tsk) |
| 290 | { |
| 291 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
| 292 | __save_init_fpu(tsk); |
| 293 | stts(); |
| 294 | } else |
| 295 | tsk->fpu_counter = 0; |
| 296 | } |
| 297 | |
| 298 | static inline void __clear_fpu(struct task_struct *tsk) |
| 299 | { |
| 300 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
Brian Gerst | 51115d4 | 2010-09-03 21:17:10 -0400 | [diff] [blame] | 301 | /* Ignore delayed exceptions from user space */ |
| 302 | asm volatile("1: fwait\n" |
| 303 | "2:\n" |
| 304 | _ASM_EXTABLE(1b, 2b)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 305 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 306 | stts(); |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | static inline void kernel_fpu_begin(void) |
| 311 | { |
| 312 | struct thread_info *me = current_thread_info(); |
| 313 | preempt_disable(); |
| 314 | if (me->status & TS_USEDFPU) |
| 315 | __save_init_fpu(me->task); |
| 316 | else |
| 317 | clts(); |
| 318 | } |
| 319 | |
| 320 | static inline void kernel_fpu_end(void) |
| 321 | { |
| 322 | stts(); |
| 323 | preempt_enable(); |
| 324 | } |
| 325 | |
Huang Ying | ae4b688 | 2009-08-31 13:11:54 +0800 | [diff] [blame] | 326 | static inline bool irq_fpu_usable(void) |
| 327 | { |
| 328 | struct pt_regs *regs; |
| 329 | |
| 330 | return !in_interrupt() || !(regs = get_irq_regs()) || \ |
| 331 | user_mode(regs) || (read_cr0() & X86_CR0_TS); |
| 332 | } |
| 333 | |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 334 | /* |
| 335 | * Some instructions like VIA's padlock instructions generate a spurious |
| 336 | * DNA fault but don't modify SSE registers. And these instructions |
Chuck Ebbert | 0b8c3d5 | 2009-06-09 10:40:50 -0400 | [diff] [blame] | 337 | * get used from interrupt context as well. To prevent these kernel instructions |
| 338 | * in interrupt context interacting wrongly with other user/kernel fpu usage, we |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 339 | * should use them only in the context of irq_ts_save/restore() |
| 340 | */ |
| 341 | static inline int irq_ts_save(void) |
| 342 | { |
| 343 | /* |
Chuck Ebbert | 0b8c3d5 | 2009-06-09 10:40:50 -0400 | [diff] [blame] | 344 | * If in process context and not atomic, we can take a spurious DNA fault. |
| 345 | * Otherwise, doing clts() in process context requires disabling preemption |
| 346 | * or some heavy lifting like kernel_fpu_begin() |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 347 | */ |
Chuck Ebbert | 0b8c3d5 | 2009-06-09 10:40:50 -0400 | [diff] [blame] | 348 | if (!in_atomic()) |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 349 | return 0; |
| 350 | |
| 351 | if (read_cr0() & X86_CR0_TS) { |
| 352 | clts(); |
| 353 | return 1; |
| 354 | } |
| 355 | |
| 356 | return 0; |
| 357 | } |
| 358 | |
| 359 | static inline void irq_ts_restore(int TS_state) |
| 360 | { |
| 361 | if (TS_state) |
| 362 | stts(); |
| 363 | } |
| 364 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 365 | /* |
| 366 | * These disable preemption on their own and are safe |
| 367 | */ |
| 368 | static inline void save_init_fpu(struct task_struct *tsk) |
| 369 | { |
| 370 | preempt_disable(); |
| 371 | __save_init_fpu(tsk); |
| 372 | stts(); |
| 373 | preempt_enable(); |
| 374 | } |
| 375 | |
| 376 | static inline void unlazy_fpu(struct task_struct *tsk) |
| 377 | { |
| 378 | preempt_disable(); |
| 379 | __unlazy_fpu(tsk); |
| 380 | preempt_enable(); |
| 381 | } |
| 382 | |
| 383 | static inline void clear_fpu(struct task_struct *tsk) |
| 384 | { |
| 385 | preempt_disable(); |
| 386 | __clear_fpu(tsk); |
| 387 | preempt_enable(); |
| 388 | } |
| 389 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 390 | /* |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 391 | * i387 state interaction |
| 392 | */ |
| 393 | static inline unsigned short get_fpu_cwd(struct task_struct *tsk) |
| 394 | { |
| 395 | if (cpu_has_fxsr) { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 396 | return tsk->thread.fpu.state->fxsave.cwd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 397 | } else { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 398 | return (unsigned short)tsk->thread.fpu.state->fsave.cwd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 399 | } |
| 400 | } |
| 401 | |
| 402 | static inline unsigned short get_fpu_swd(struct task_struct *tsk) |
| 403 | { |
| 404 | if (cpu_has_fxsr) { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 405 | return tsk->thread.fpu.state->fxsave.swd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 406 | } else { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 407 | return (unsigned short)tsk->thread.fpu.state->fsave.swd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 408 | } |
| 409 | } |
| 410 | |
| 411 | static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) |
| 412 | { |
| 413 | if (cpu_has_xmm) { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 414 | return tsk->thread.fpu.state->fxsave.mxcsr; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 415 | } else { |
| 416 | return MXCSR_DEFAULT; |
| 417 | } |
| 418 | } |
| 419 | |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 420 | static bool fpu_allocated(struct fpu *fpu) |
| 421 | { |
| 422 | return fpu->state != NULL; |
| 423 | } |
| 424 | |
| 425 | static inline int fpu_alloc(struct fpu *fpu) |
| 426 | { |
| 427 | if (fpu_allocated(fpu)) |
| 428 | return 0; |
| 429 | fpu->state = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL); |
| 430 | if (!fpu->state) |
| 431 | return -ENOMEM; |
| 432 | WARN_ON((unsigned long)fpu->state & 15); |
| 433 | return 0; |
| 434 | } |
| 435 | |
| 436 | static inline void fpu_free(struct fpu *fpu) |
| 437 | { |
| 438 | if (fpu->state) { |
| 439 | kmem_cache_free(task_xstate_cachep, fpu->state); |
| 440 | fpu->state = NULL; |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | static inline void fpu_copy(struct fpu *dst, struct fpu *src) |
| 445 | { |
| 446 | memcpy(dst->state, src->state, xstate_size); |
| 447 | } |
| 448 | |
Sheng Yang | 5ee481d | 2010-05-17 17:22:23 +0800 | [diff] [blame] | 449 | extern void fpu_finit(struct fpu *fpu); |
| 450 | |
Herbert Xu | 3b0d659 | 2009-11-03 09:11:15 -0500 | [diff] [blame] | 451 | #endif /* __ASSEMBLY__ */ |
| 452 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 453 | #endif /* _ASM_X86_I387_H */ |