blob: fa984b53e7e6c8e0d0118b5ef195b276c4adfcc7 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86_64/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 */
8
9/*
10 * entry.S contains the system-call and fault low-level handling routines.
11 *
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call.
14 *
15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al.
17 *
18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved.
Andi Kleen2e91a172006-09-26 10:52:29 +020023 *
24 * Some macro usage:
25 * - CFI macros are used to generate dwarf2 unwind information for better
26 * backtraces. They don't change any code.
27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29 * There are unfortunately lots of special cases where some registers
30 * not touched. The macro is a big mess that should be cleaned up.
31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32 * Gives a full stack frame.
33 * - ENTRY/END Define functions in the symbol table.
34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35 * frame that is otherwise undefined after a SYSCALL
36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
Linus Torvalds1da177e2005-04-16 15:20:36 -070038 */
39
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#include <linux/linkage.h>
41#include <asm/segment.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070042#include <asm/cache.h>
43#include <asm/errno.h>
44#include <asm/dwarf2.h>
45#include <asm/calling.h>
Sam Ravnborge2d5df92005-09-09 21:28:48 +020046#include <asm/asm-offsets.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#include <asm/msr.h>
48#include <asm/unistd.h>
49#include <asm/thread_info.h>
50#include <asm/hw_irq.h>
Andi Kleen5f8efbb2006-01-16 01:56:39 +010051#include <asm/page.h>
Ingo Molnar2601e642006-07-03 00:24:45 -070052#include <asm/irqflags.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070053
54 .code64
55
Andi Kleendc37db42005-04-16 15:25:05 -070056#ifndef CONFIG_PREEMPT
Linus Torvalds1da177e2005-04-16 15:20:36 -070057#define retint_kernel retint_restore_args
58#endif
Ingo Molnar2601e642006-07-03 00:24:45 -070059
60
61.macro TRACE_IRQS_IRETQ offset=ARGOFFSET
62#ifdef CONFIG_TRACE_IRQFLAGS
63 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
64 jnc 1f
65 TRACE_IRQS_ON
661:
67#endif
68.endm
69
Linus Torvalds1da177e2005-04-16 15:20:36 -070070/*
71 * C code is not supposed to know about undefined top of stack. Every time
72 * a C function with an pt_regs argument is called from the SYSCALL based
73 * fast path FIXUP_TOP_OF_STACK is needed.
74 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
75 * manipulation.
76 */
77
78 /* %rsp:at FRAMEEND */
79 .macro FIXUP_TOP_OF_STACK tmp
80 movq %gs:pda_oldrsp,\tmp
81 movq \tmp,RSP(%rsp)
82 movq $__USER_DS,SS(%rsp)
83 movq $__USER_CS,CS(%rsp)
84 movq $-1,RCX(%rsp)
85 movq R11(%rsp),\tmp /* get eflags */
86 movq \tmp,EFLAGS(%rsp)
87 .endm
88
89 .macro RESTORE_TOP_OF_STACK tmp,offset=0
90 movq RSP-\offset(%rsp),\tmp
91 movq \tmp,%gs:pda_oldrsp
92 movq EFLAGS-\offset(%rsp),\tmp
93 movq \tmp,R11-\offset(%rsp)
94 .endm
95
96 .macro FAKE_STACK_FRAME child_rip
97 /* push in order ss, rsp, eflags, cs, rip */
Andi Kleen3829ee62005-07-28 21:15:48 -070098 xorl %eax, %eax
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 pushq %rax /* ss */
100 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich7effaa82005-09-12 18:49:24 +0200101 /*CFI_REL_OFFSET ss,0*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 pushq %rax /* rsp */
103 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich7effaa82005-09-12 18:49:24 +0200104 CFI_REL_OFFSET rsp,0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 pushq $(1<<9) /* eflags - interrupts on */
106 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich7effaa82005-09-12 18:49:24 +0200107 /*CFI_REL_OFFSET rflags,0*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 pushq $__KERNEL_CS /* cs */
109 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich7effaa82005-09-12 18:49:24 +0200110 /*CFI_REL_OFFSET cs,0*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 pushq \child_rip /* rip */
112 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich7effaa82005-09-12 18:49:24 +0200113 CFI_REL_OFFSET rip,0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 pushq %rax /* orig rax */
115 CFI_ADJUST_CFA_OFFSET 8
116 .endm
117
118 .macro UNFAKE_STACK_FRAME
119 addq $8*6, %rsp
120 CFI_ADJUST_CFA_OFFSET -(6*8)
121 .endm
122
Jan Beulich7effaa82005-09-12 18:49:24 +0200123 .macro CFI_DEFAULT_STACK start=1
124 .if \start
125 CFI_STARTPROC simple
Jan Beulichadf14232006-09-26 10:52:41 +0200126 CFI_SIGNAL_FRAME
Jan Beulich7effaa82005-09-12 18:49:24 +0200127 CFI_DEF_CFA rsp,SS+8
128 .else
129 CFI_DEF_CFA_OFFSET SS+8
130 .endif
131 CFI_REL_OFFSET r15,R15
132 CFI_REL_OFFSET r14,R14
133 CFI_REL_OFFSET r13,R13
134 CFI_REL_OFFSET r12,R12
135 CFI_REL_OFFSET rbp,RBP
136 CFI_REL_OFFSET rbx,RBX
137 CFI_REL_OFFSET r11,R11
138 CFI_REL_OFFSET r10,R10
139 CFI_REL_OFFSET r9,R9
140 CFI_REL_OFFSET r8,R8
141 CFI_REL_OFFSET rax,RAX
142 CFI_REL_OFFSET rcx,RCX
143 CFI_REL_OFFSET rdx,RDX
144 CFI_REL_OFFSET rsi,RSI
145 CFI_REL_OFFSET rdi,RDI
146 CFI_REL_OFFSET rip,RIP
147 /*CFI_REL_OFFSET cs,CS*/
148 /*CFI_REL_OFFSET rflags,EFLAGS*/
149 CFI_REL_OFFSET rsp,RSP
150 /*CFI_REL_OFFSET ss,SS*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151 .endm
152/*
153 * A newly forked process directly context switches into this.
154 */
155/* rdi: prev */
156ENTRY(ret_from_fork)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 CFI_DEFAULT_STACK
Andi Kleen658fdbe2006-09-26 10:52:41 +0200158 push kernel_eflags(%rip)
159 CFI_ADJUST_CFA_OFFSET 4
160 popf # reset kernel eflags
161 CFI_ADJUST_CFA_OFFSET -4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 call schedule_tail
163 GET_THREAD_INFO(%rcx)
164 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
165 jnz rff_trace
166rff_action:
167 RESTORE_REST
168 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
169 je int_ret_from_sys_call
170 testl $_TIF_IA32,threadinfo_flags(%rcx)
171 jnz int_ret_from_sys_call
172 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
173 jmp ret_from_sys_call
174rff_trace:
175 movq %rsp,%rdi
176 call syscall_trace_leave
177 GET_THREAD_INFO(%rcx)
178 jmp rff_action
179 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +0200180END(ret_from_fork)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
182/*
183 * System call entry. Upto 6 arguments in registers are supported.
184 *
185 * SYSCALL does not save anything on the stack and does not change the
186 * stack pointer.
187 */
188
189/*
190 * Register setup:
191 * rax system call number
192 * rdi arg0
193 * rcx return address for syscall/sysret, C arg3
194 * rsi arg1
195 * rdx arg2
196 * r10 arg3 (--> moved to rcx for C)
197 * r8 arg4
198 * r9 arg5
199 * r11 eflags for syscall/sysret, temporary for C
200 * r12-r15,rbp,rbx saved by C code, not touched.
201 *
202 * Interrupts are off on entry.
203 * Only called from user space.
204 *
205 * XXX if we had a free scratch register we could save the RSP into the stack frame
206 * and report it properly in ps. Unfortunately we haven't.
Andi Kleen7bf36bb2006-04-07 19:50:00 +0200207 *
208 * When user can change the frames always force IRET. That is because
209 * it deals with uncanonical addresses better. SYSRET has trouble
210 * with them due to bugs in both AMD and Intel CPUs.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 */
212
213ENTRY(system_call)
Jan Beulich7effaa82005-09-12 18:49:24 +0200214 CFI_STARTPROC simple
Jan Beulichadf14232006-09-26 10:52:41 +0200215 CFI_SIGNAL_FRAME
Jan Beulichdffead42006-06-26 13:57:38 +0200216 CFI_DEF_CFA rsp,PDA_STACKOFFSET
Jan Beulich7effaa82005-09-12 18:49:24 +0200217 CFI_REGISTER rip,rcx
218 /*CFI_REGISTER rflags,r11*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219 swapgs
220 movq %rsp,%gs:pda_oldrsp
221 movq %gs:pda_kernelstack,%rsp
Ingo Molnar2601e642006-07-03 00:24:45 -0700222 /*
223 * No need to follow this irqs off/on section - it's straight
224 * and short:
225 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226 sti
227 SAVE_ARGS 8,1
228 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
Jan Beulich7effaa82005-09-12 18:49:24 +0200229 movq %rcx,RIP-ARGOFFSET(%rsp)
230 CFI_REL_OFFSET rip,RIP-ARGOFFSET
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 GET_THREAD_INFO(%rcx)
232 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
233 jnz tracesys
234 cmpq $__NR_syscall_max,%rax
235 ja badsys
236 movq %r10,%rcx
237 call *sys_call_table(,%rax,8) # XXX: rip relative
238 movq %rax,RAX-ARGOFFSET(%rsp)
239/*
240 * Syscall return path ending with SYSRET (fast path)
241 * Has incomplete stack frame and undefined top of stack.
242 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243ret_from_sys_call:
Andi Kleen11b854b2005-04-16 15:25:02 -0700244 movl $_TIF_ALLWORK_MASK,%edi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 /* edi: flagmask */
246sysret_check:
247 GET_THREAD_INFO(%rcx)
248 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700249 TRACE_IRQS_OFF
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 movl threadinfo_flags(%rcx),%edx
251 andl %edi,%edx
252 jnz sysret_careful
Jan Beulichbcddc012006-12-07 02:14:02 +0100253 CFI_REMEMBER_STATE
Ingo Molnar2601e642006-07-03 00:24:45 -0700254 /*
255 * sysretq will re-enable interrupts:
256 */
257 TRACE_IRQS_ON
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 movq RIP-ARGOFFSET(%rsp),%rcx
Jan Beulich7effaa82005-09-12 18:49:24 +0200259 CFI_REGISTER rip,rcx
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 RESTORE_ARGS 0,-ARG_SKIP,1
Jan Beulich7effaa82005-09-12 18:49:24 +0200261 /*CFI_REGISTER rflags,r11*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 movq %gs:pda_oldrsp,%rsp
263 swapgs
264 sysretq
265
Jan Beulichbcddc012006-12-07 02:14:02 +0100266 CFI_RESTORE_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 /* Handle reschedules */
268 /* edx: work, edi: workmask */
269sysret_careful:
270 bt $TIF_NEED_RESCHED,%edx
271 jnc sysret_signal
Ingo Molnar2601e642006-07-03 00:24:45 -0700272 TRACE_IRQS_ON
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273 sti
274 pushq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200275 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 call schedule
277 popq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200278 CFI_ADJUST_CFA_OFFSET -8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 jmp sysret_check
280
281 /* Handle a signal */
282sysret_signal:
Ingo Molnar2601e642006-07-03 00:24:45 -0700283 TRACE_IRQS_ON
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 sti
Andi Kleen10ffdbb2005-05-16 21:53:19 -0700285 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
286 jz 1f
287
288 /* Really a signal */
289 /* edx: work flags (arg3) */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290 leaq do_notify_resume(%rip),%rax
291 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
292 xorl %esi,%esi # oldset -> arg2
293 call ptregscall_common
Andi Kleen10ffdbb2005-05-16 21:53:19 -07002941: movl $_TIF_NEED_RESCHED,%edi
Andi Kleen7bf36bb2006-04-07 19:50:00 +0200295 /* Use IRET because user could have changed frame. This
296 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
297 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700298 TRACE_IRQS_OFF
Andi Kleen7bf36bb2006-04-07 19:50:00 +0200299 jmp int_with_check
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300
Jan Beulich7effaa82005-09-12 18:49:24 +0200301badsys:
302 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
303 jmp ret_from_sys_call
304
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 /* Do syscall tracing */
306tracesys:
307 SAVE_REST
308 movq $-ENOSYS,RAX(%rsp)
309 FIXUP_TOP_OF_STACK %rdi
310 movq %rsp,%rdi
311 call syscall_trace_enter
312 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
313 RESTORE_REST
314 cmpq $__NR_syscall_max,%rax
Jan Beulichcc7d4792006-10-21 18:37:02 +0200315 movq $-ENOSYS,%rcx
316 cmova %rcx,%rax
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 ja 1f
318 movq %r10,%rcx /* fixup for C */
319 call *sys_call_table(,%rax,8)
Andi Kleen822ff012006-05-30 22:48:03 +02003201: movq %rax,RAX-ARGOFFSET(%rsp)
Andi Kleen7bf36bb2006-04-07 19:50:00 +0200321 /* Use IRET because user could have changed frame */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323/*
324 * Syscall return path ending with IRET.
325 * Has correct top of stack, but partial stack frame.
Jan Beulichbcddc012006-12-07 02:14:02 +0100326 */
327 .globl int_ret_from_sys_call
328int_ret_from_sys_call:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700330 TRACE_IRQS_OFF
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 testl $3,CS-ARGOFFSET(%rsp)
332 je retint_restore_args
333 movl $_TIF_ALLWORK_MASK,%edi
334 /* edi: mask to check */
335int_with_check:
336 GET_THREAD_INFO(%rcx)
337 movl threadinfo_flags(%rcx),%edx
338 andl %edi,%edx
339 jnz int_careful
Andi Kleenbf2fcc62006-01-11 22:44:06 +0100340 andl $~TS_COMPAT,threadinfo_status(%rcx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 jmp retint_swapgs
342
343 /* Either reschedule or signal or syscall exit tracking needed. */
344 /* First do a reschedule test. */
345 /* edx: work, edi: workmask */
346int_careful:
347 bt $TIF_NEED_RESCHED,%edx
348 jnc int_very_careful
Ingo Molnar2601e642006-07-03 00:24:45 -0700349 TRACE_IRQS_ON
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 sti
351 pushq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200352 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 call schedule
354 popq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200355 CFI_ADJUST_CFA_OFFSET -8
Andi Kleencdd219c2005-04-16 15:25:04 -0700356 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700357 TRACE_IRQS_OFF
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 jmp int_with_check
359
360 /* handle signals and tracing -- both require a full stack frame */
361int_very_careful:
Ingo Molnar2601e642006-07-03 00:24:45 -0700362 TRACE_IRQS_ON
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 sti
364 SAVE_REST
365 /* Check for syscall exit trace */
366 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
367 jz int_signal
368 pushq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200369 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 leaq 8(%rsp),%rdi # &ptregs -> arg1
371 call syscall_trace_leave
372 popq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200373 CFI_ADJUST_CFA_OFFSET -8
Andi Kleen36c11042005-04-16 15:25:01 -0700374 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375 jmp int_restore_rest
376
377int_signal:
378 testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
379 jz 1f
380 movq %rsp,%rdi # &ptregs -> arg1
381 xorl %esi,%esi # oldset -> arg2
382 call do_notify_resume
3831: movl $_TIF_NEED_RESCHED,%edi
384int_restore_rest:
385 RESTORE_REST
Andi Kleenbe9e6872005-05-01 08:58:51 -0700386 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700387 TRACE_IRQS_OFF
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 jmp int_with_check
389 CFI_ENDPROC
Jan Beulichbcddc012006-12-07 02:14:02 +0100390END(system_call)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391
392/*
393 * Certain special system calls that need to save a complete full stack frame.
394 */
395
396 .macro PTREGSCALL label,func,arg
397 .globl \label
398\label:
399 leaq \func(%rip),%rax
400 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
401 jmp ptregscall_common
Jan Beulich4b787e02006-06-26 13:56:55 +0200402END(\label)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403 .endm
404
Jan Beulich7effaa82005-09-12 18:49:24 +0200405 CFI_STARTPROC
406
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 PTREGSCALL stub_clone, sys_clone, %r8
408 PTREGSCALL stub_fork, sys_fork, %rdi
409 PTREGSCALL stub_vfork, sys_vfork, %rdi
410 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
411 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
412 PTREGSCALL stub_iopl, sys_iopl, %rsi
413
414ENTRY(ptregscall_common)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 popq %r11
Jan Beulich7effaa82005-09-12 18:49:24 +0200416 CFI_ADJUST_CFA_OFFSET -8
417 CFI_REGISTER rip, r11
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418 SAVE_REST
419 movq %r11, %r15
Jan Beulich7effaa82005-09-12 18:49:24 +0200420 CFI_REGISTER rip, r15
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 FIXUP_TOP_OF_STACK %r11
422 call *%rax
423 RESTORE_TOP_OF_STACK %r11
424 movq %r15, %r11
Jan Beulich7effaa82005-09-12 18:49:24 +0200425 CFI_REGISTER rip, r11
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 RESTORE_REST
427 pushq %r11
Jan Beulich7effaa82005-09-12 18:49:24 +0200428 CFI_ADJUST_CFA_OFFSET 8
429 CFI_REL_OFFSET rip, 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 ret
431 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +0200432END(ptregscall_common)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433
434ENTRY(stub_execve)
435 CFI_STARTPROC
436 popq %r11
Jan Beulich7effaa82005-09-12 18:49:24 +0200437 CFI_ADJUST_CFA_OFFSET -8
438 CFI_REGISTER rip, r11
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 SAVE_REST
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 FIXUP_TOP_OF_STACK %r11
441 call sys_execve
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 RESTORE_TOP_OF_STACK %r11
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 movq %rax,RAX(%rsp)
444 RESTORE_REST
445 jmp int_ret_from_sys_call
446 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +0200447END(stub_execve)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448
449/*
450 * sigreturn is special because it needs to restore all registers on return.
451 * This cannot be done with SYSRET, so use the IRET return path instead.
452 */
453ENTRY(stub_rt_sigreturn)
454 CFI_STARTPROC
Jan Beulich7effaa82005-09-12 18:49:24 +0200455 addq $8, %rsp
456 CFI_ADJUST_CFA_OFFSET -8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457 SAVE_REST
458 movq %rsp,%rdi
459 FIXUP_TOP_OF_STACK %r11
460 call sys_rt_sigreturn
461 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
462 RESTORE_REST
463 jmp int_ret_from_sys_call
464 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +0200465END(stub_rt_sigreturn)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466
Jan Beulich7effaa82005-09-12 18:49:24 +0200467/*
468 * initial frame state for interrupts and exceptions
469 */
470 .macro _frame ref
471 CFI_STARTPROC simple
Jan Beulichadf14232006-09-26 10:52:41 +0200472 CFI_SIGNAL_FRAME
Jan Beulich7effaa82005-09-12 18:49:24 +0200473 CFI_DEF_CFA rsp,SS+8-\ref
474 /*CFI_REL_OFFSET ss,SS-\ref*/
475 CFI_REL_OFFSET rsp,RSP-\ref
476 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
477 /*CFI_REL_OFFSET cs,CS-\ref*/
478 CFI_REL_OFFSET rip,RIP-\ref
479 .endm
480
481/* initial frame state for interrupts (and exceptions without error code) */
482#define INTR_FRAME _frame RIP
483/* initial frame state for exceptions with error code (and interrupts with
484 vector already pushed) */
485#define XCPT_FRAME _frame ORIG_RAX
486
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487/*
488 * Interrupt entry/exit.
489 *
490 * Interrupt entry points save only callee clobbered registers in fast path.
491 *
492 * Entry runs with interrupts off.
493 */
494
495/* 0(%rsp): interrupt number */
496 .macro interrupt func
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 cld
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498 SAVE_ARGS
499 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
Jan Beulich1de9c3f2006-06-26 13:57:35 +0200500 pushq %rbp
501 CFI_ADJUST_CFA_OFFSET 8
502 CFI_REL_OFFSET rbp, 0
503 movq %rsp,%rbp
504 CFI_DEF_CFA_REGISTER rbp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 testl $3,CS(%rdi)
506 je 1f
507 swapgs
Andi Kleen96e54042006-09-26 10:52:39 +0200508 /* irqcount is used to check if a CPU is already on an interrupt
509 stack or not. While this is essentially redundant with preempt_count
510 it is a little cheaper to use a separate counter in the PDA
511 (short of moving irq_enter into assembly, which would be too
512 much work) */
5131: incl %gs:pda_irqcount
Jan Beulich1de9c3f2006-06-26 13:57:35 +0200514 cmoveq %gs:pda_irqstackptr,%rsp
Andi Kleen26995002006-08-02 22:37:28 +0200515 push %rbp # backlink for old unwinder
Ingo Molnar2601e642006-07-03 00:24:45 -0700516 /*
517 * We entered an interrupt context - irqs are off:
518 */
519 TRACE_IRQS_OFF
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 call \func
521 .endm
522
523ENTRY(common_interrupt)
Jan Beulich7effaa82005-09-12 18:49:24 +0200524 XCPT_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 interrupt do_IRQ
526 /* 0(%rsp): oldrsp-ARGOFFSET */
Jan Beulich7effaa82005-09-12 18:49:24 +0200527ret_from_intr:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700529 TRACE_IRQS_OFF
Andi Kleen3829ee62005-07-28 21:15:48 -0700530 decl %gs:pda_irqcount
Jan Beulich1de9c3f2006-06-26 13:57:35 +0200531 leaveq
Jan Beulich7effaa82005-09-12 18:49:24 +0200532 CFI_DEF_CFA_REGISTER rsp
Jan Beulich1de9c3f2006-06-26 13:57:35 +0200533 CFI_ADJUST_CFA_OFFSET -8
Jan Beulich7effaa82005-09-12 18:49:24 +0200534exit_intr:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 GET_THREAD_INFO(%rcx)
536 testl $3,CS-ARGOFFSET(%rsp)
537 je retint_kernel
538
539 /* Interrupt came from user space */
540 /*
541 * Has a correct top of stack, but a partial stack frame
542 * %rcx: thread info. Interrupts off.
543 */
544retint_with_reschedule:
545 movl $_TIF_WORK_MASK,%edi
Jan Beulich7effaa82005-09-12 18:49:24 +0200546retint_check:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 movl threadinfo_flags(%rcx),%edx
548 andl %edi,%edx
Jan Beulich7effaa82005-09-12 18:49:24 +0200549 CFI_REMEMBER_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 jnz retint_careful
551retint_swapgs:
Ingo Molnar2601e642006-07-03 00:24:45 -0700552 /*
553 * The iretq could re-enable interrupts:
554 */
555 cli
556 TRACE_IRQS_IRETQ
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 swapgs
Ingo Molnar2601e642006-07-03 00:24:45 -0700558 jmp restore_args
559
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560retint_restore_args:
561 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700562 /*
563 * The iretq could re-enable interrupts:
564 */
565 TRACE_IRQS_IRETQ
566restore_args:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567 RESTORE_ARGS 0,8,0
568iret_label:
569 iretq
570
571 .section __ex_table,"a"
572 .quad iret_label,bad_iret
573 .previous
574 .section .fixup,"ax"
575 /* force a signal here? this matches i386 behaviour */
576 /* running with kernel gs */
577bad_iret:
Andi Kleen3076a492006-03-25 16:31:55 +0100578 movq $11,%rdi /* SIGSEGV */
Ingo Molnar2601e642006-07-03 00:24:45 -0700579 TRACE_IRQS_ON
Andi Kleen2391c4b2006-02-16 23:42:01 +0100580 sti
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 jmp do_exit
582 .previous
583
Jan Beulich7effaa82005-09-12 18:49:24 +0200584 /* edi: workmask, edx: work */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585retint_careful:
Jan Beulich7effaa82005-09-12 18:49:24 +0200586 CFI_RESTORE_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 bt $TIF_NEED_RESCHED,%edx
588 jnc retint_signal
Ingo Molnar2601e642006-07-03 00:24:45 -0700589 TRACE_IRQS_ON
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 sti
591 pushq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200592 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 call schedule
594 popq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200595 CFI_ADJUST_CFA_OFFSET -8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 GET_THREAD_INFO(%rcx)
597 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700598 TRACE_IRQS_OFF
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599 jmp retint_check
600
601retint_signal:
Andi Kleen10ffdbb2005-05-16 21:53:19 -0700602 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
603 jz retint_swapgs
Ingo Molnar2601e642006-07-03 00:24:45 -0700604 TRACE_IRQS_ON
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 sti
606 SAVE_REST
607 movq $-1,ORIG_RAX(%rsp)
Andi Kleen3829ee62005-07-28 21:15:48 -0700608 xorl %esi,%esi # oldset
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 movq %rsp,%rdi # &pt_regs
610 call do_notify_resume
611 RESTORE_REST
612 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700613 TRACE_IRQS_OFF
Andi Kleen10ffdbb2005-05-16 21:53:19 -0700614 movl $_TIF_NEED_RESCHED,%edi
Andi Kleenbe9e6872005-05-01 08:58:51 -0700615 GET_THREAD_INFO(%rcx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 jmp retint_check
617
618#ifdef CONFIG_PREEMPT
619 /* Returning to kernel space. Check if we need preemption */
620 /* rcx: threadinfo. interrupts off. */
Andi Kleenb06baba2006-09-26 10:52:29 +0200621ENTRY(retint_kernel)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 cmpl $0,threadinfo_preempt_count(%rcx)
623 jnz retint_restore_args
624 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
625 jnc retint_restore_args
626 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
627 jnc retint_restore_args
628 call preempt_schedule_irq
629 jmp exit_intr
630#endif
Jan Beulich4b787e02006-06-26 13:56:55 +0200631
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +0200633END(common_interrupt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634
635/*
636 * APIC interrupts.
637 */
638 .macro apicinterrupt num,func
Jan Beulich7effaa82005-09-12 18:49:24 +0200639 INTR_FRAME
Rusty Russell19eadf92006-06-27 02:53:44 -0700640 pushq $~(\num)
Jan Beulich7effaa82005-09-12 18:49:24 +0200641 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 interrupt \func
643 jmp ret_from_intr
644 CFI_ENDPROC
645 .endm
646
647ENTRY(thermal_interrupt)
648 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
Jan Beulich4b787e02006-06-26 13:56:55 +0200649END(thermal_interrupt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650
Jacob Shin89b831e2005-11-05 17:25:53 +0100651ENTRY(threshold_interrupt)
652 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
Jan Beulich4b787e02006-06-26 13:56:55 +0200653END(threshold_interrupt)
Jacob Shin89b831e2005-11-05 17:25:53 +0100654
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655#ifdef CONFIG_SMP
656ENTRY(reschedule_interrupt)
657 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
Jan Beulich4b787e02006-06-26 13:56:55 +0200658END(reschedule_interrupt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659
Andi Kleene5bc8b62005-09-12 18:49:24 +0200660 .macro INVALIDATE_ENTRY num
661ENTRY(invalidate_interrupt\num)
662 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
Jan Beulich4b787e02006-06-26 13:56:55 +0200663END(invalidate_interrupt\num)
Andi Kleene5bc8b62005-09-12 18:49:24 +0200664 .endm
665
666 INVALIDATE_ENTRY 0
667 INVALIDATE_ENTRY 1
668 INVALIDATE_ENTRY 2
669 INVALIDATE_ENTRY 3
670 INVALIDATE_ENTRY 4
671 INVALIDATE_ENTRY 5
672 INVALIDATE_ENTRY 6
673 INVALIDATE_ENTRY 7
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674
675ENTRY(call_function_interrupt)
676 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
Jan Beulich4b787e02006-06-26 13:56:55 +0200677END(call_function_interrupt)
Eric W. Biederman61014292007-02-23 04:40:58 -0700678ENTRY(irq_move_cleanup_interrupt)
679 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
680END(irq_move_cleanup_interrupt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681#endif
682
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683ENTRY(apic_timer_interrupt)
684 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
Jan Beulich4b787e02006-06-26 13:56:55 +0200685END(apic_timer_interrupt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686
687ENTRY(error_interrupt)
688 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
Jan Beulich4b787e02006-06-26 13:56:55 +0200689END(error_interrupt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690
691ENTRY(spurious_interrupt)
692 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
Jan Beulich4b787e02006-06-26 13:56:55 +0200693END(spurious_interrupt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694
695/*
696 * Exception entry points.
697 */
698 .macro zeroentry sym
Jan Beulich7effaa82005-09-12 18:49:24 +0200699 INTR_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 pushq $0 /* push error code/oldrax */
Jan Beulich7effaa82005-09-12 18:49:24 +0200701 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 pushq %rax /* push real oldrax to the rdi slot */
Jan Beulich7effaa82005-09-12 18:49:24 +0200703 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich37550902007-05-02 19:27:05 +0200704 CFI_REL_OFFSET rax,0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 leaq \sym(%rip),%rax
706 jmp error_entry
Jan Beulich7effaa82005-09-12 18:49:24 +0200707 CFI_ENDPROC
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708 .endm
709
710 .macro errorentry sym
Jan Beulich7effaa82005-09-12 18:49:24 +0200711 XCPT_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712 pushq %rax
Jan Beulich7effaa82005-09-12 18:49:24 +0200713 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich37550902007-05-02 19:27:05 +0200714 CFI_REL_OFFSET rax,0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715 leaq \sym(%rip),%rax
716 jmp error_entry
Jan Beulich7effaa82005-09-12 18:49:24 +0200717 CFI_ENDPROC
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 .endm
719
720 /* error code is on the stack already */
721 /* handle NMI like exceptions that can happen everywhere */
Ingo Molnar2601e642006-07-03 00:24:45 -0700722 .macro paranoidentry sym, ist=0, irqtrace=1
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 SAVE_ALL
724 cld
725 movl $1,%ebx
726 movl $MSR_GS_BASE,%ecx
727 rdmsr
728 testl %edx,%edx
729 js 1f
730 swapgs
731 xorl %ebx,%ebx
Jan Beulichb556b352006-01-11 22:43:00 +01007321:
733 .if \ist
734 movq %gs:pda_data_offset, %rbp
735 .endif
736 movq %rsp,%rdi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737 movq ORIG_RAX(%rsp),%rsi
738 movq $-1,ORIG_RAX(%rsp)
Jan Beulichb556b352006-01-11 22:43:00 +0100739 .if \ist
Andi Kleen5f8efbb2006-01-16 01:56:39 +0100740 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
Jan Beulichb556b352006-01-11 22:43:00 +0100741 .endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742 call \sym
Jan Beulichb556b352006-01-11 22:43:00 +0100743 .if \ist
Andi Kleen5f8efbb2006-01-16 01:56:39 +0100744 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
Jan Beulichb556b352006-01-11 22:43:00 +0100745 .endif
Andi Kleen6fefb0d2005-04-16 15:25:03 -0700746 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700747 .if \irqtrace
748 TRACE_IRQS_OFF
749 .endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750 .endm
Ingo Molnar2601e642006-07-03 00:24:45 -0700751
752 /*
753 * "Paranoid" exit path from exception stack.
754 * Paranoid because this is used by NMIs and cannot take
755 * any kernel state for granted.
756 * We don't do kernel preemption checks here, because only
757 * NMI should be common and it does not enable IRQs and
758 * cannot get reschedule ticks.
759 *
760 * "trace" is 0 for the NMI handler only, because irq-tracing
761 * is fundamentally NMI-unsafe. (we cannot change the soft and
762 * hard flags at once, atomically)
763 */
764 .macro paranoidexit trace=1
765 /* ebx: no swapgs flag */
766paranoid_exit\trace:
767 testl %ebx,%ebx /* swapgs needed? */
768 jnz paranoid_restore\trace
769 testl $3,CS(%rsp)
770 jnz paranoid_userspace\trace
771paranoid_swapgs\trace:
Andi Kleen7a0a2df2006-09-26 10:52:37 +0200772 .if \trace
Ingo Molnar2601e642006-07-03 00:24:45 -0700773 TRACE_IRQS_IRETQ 0
Andi Kleen7a0a2df2006-09-26 10:52:37 +0200774 .endif
Ingo Molnar2601e642006-07-03 00:24:45 -0700775 swapgs
776paranoid_restore\trace:
777 RESTORE_ALL 8
778 iretq
779paranoid_userspace\trace:
780 GET_THREAD_INFO(%rcx)
781 movl threadinfo_flags(%rcx),%ebx
782 andl $_TIF_WORK_MASK,%ebx
783 jz paranoid_swapgs\trace
784 movq %rsp,%rdi /* &pt_regs */
785 call sync_regs
786 movq %rax,%rsp /* switch stack for scheduling */
787 testl $_TIF_NEED_RESCHED,%ebx
788 jnz paranoid_schedule\trace
789 movl %ebx,%edx /* arg3: thread flags */
790 .if \trace
791 TRACE_IRQS_ON
792 .endif
793 sti
794 xorl %esi,%esi /* arg2: oldset */
795 movq %rsp,%rdi /* arg1: &pt_regs */
796 call do_notify_resume
797 cli
798 .if \trace
799 TRACE_IRQS_OFF
800 .endif
801 jmp paranoid_userspace\trace
802paranoid_schedule\trace:
803 .if \trace
804 TRACE_IRQS_ON
805 .endif
806 sti
807 call schedule
808 cli
809 .if \trace
810 TRACE_IRQS_OFF
811 .endif
812 jmp paranoid_userspace\trace
813 CFI_ENDPROC
814 .endm
815
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816/*
817 * Exception entry point. This expects an error code/orig_rax on the stack
818 * and the exception handler in %rax.
819 */
Prasanna S.Pd28c4392006-09-26 10:52:34 +0200820KPROBE_ENTRY(error_entry)
Jan Beulich7effaa82005-09-12 18:49:24 +0200821 _frame RDI
Jan Beulich37550902007-05-02 19:27:05 +0200822 CFI_REL_OFFSET rax,0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 /* rdi slot contains rax, oldrax contains error code */
824 cld
825 subq $14*8,%rsp
826 CFI_ADJUST_CFA_OFFSET (14*8)
827 movq %rsi,13*8(%rsp)
828 CFI_REL_OFFSET rsi,RSI
829 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
Jan Beulich37550902007-05-02 19:27:05 +0200830 CFI_REGISTER rax,rsi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831 movq %rdx,12*8(%rsp)
832 CFI_REL_OFFSET rdx,RDX
833 movq %rcx,11*8(%rsp)
834 CFI_REL_OFFSET rcx,RCX
835 movq %rsi,10*8(%rsp) /* store rax */
836 CFI_REL_OFFSET rax,RAX
837 movq %r8, 9*8(%rsp)
838 CFI_REL_OFFSET r8,R8
839 movq %r9, 8*8(%rsp)
840 CFI_REL_OFFSET r9,R9
841 movq %r10,7*8(%rsp)
842 CFI_REL_OFFSET r10,R10
843 movq %r11,6*8(%rsp)
844 CFI_REL_OFFSET r11,R11
845 movq %rbx,5*8(%rsp)
846 CFI_REL_OFFSET rbx,RBX
847 movq %rbp,4*8(%rsp)
848 CFI_REL_OFFSET rbp,RBP
849 movq %r12,3*8(%rsp)
850 CFI_REL_OFFSET r12,R12
851 movq %r13,2*8(%rsp)
852 CFI_REL_OFFSET r13,R13
853 movq %r14,1*8(%rsp)
854 CFI_REL_OFFSET r14,R14
855 movq %r15,(%rsp)
856 CFI_REL_OFFSET r15,R15
857 xorl %ebx,%ebx
858 testl $3,CS(%rsp)
859 je error_kernelspace
860error_swapgs:
861 swapgs
862error_sti:
863 movq %rdi,RDI(%rsp)
Jan Beulich37550902007-05-02 19:27:05 +0200864 CFI_REL_OFFSET rdi,RDI
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865 movq %rsp,%rdi
866 movq ORIG_RAX(%rsp),%rsi /* get error code */
867 movq $-1,ORIG_RAX(%rsp)
868 call *%rax
869 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
870error_exit:
871 movl %ebx,%eax
872 RESTORE_REST
873 cli
Ingo Molnar2601e642006-07-03 00:24:45 -0700874 TRACE_IRQS_OFF
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875 GET_THREAD_INFO(%rcx)
876 testl %eax,%eax
877 jne retint_kernel
878 movl threadinfo_flags(%rcx),%edx
879 movl $_TIF_WORK_MASK,%edi
880 andl %edi,%edx
881 jnz retint_careful
Ingo Molnar2601e642006-07-03 00:24:45 -0700882 /*
883 * The iret might restore flags:
884 */
885 TRACE_IRQS_IRETQ
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 swapgs
887 RESTORE_ARGS 0,8,0
Jan Beulich505cc4e2006-01-11 22:42:20 +0100888 jmp iret_label
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 CFI_ENDPROC
890
891error_kernelspace:
892 incl %ebx
893 /* There are two places in the kernel that can potentially fault with
894 usergs. Handle them here. The exception handlers after
895 iret run with kernel gs again, so don't set the user space flag.
896 B stepping K8s sometimes report an truncated RIP for IRET
897 exceptions returning to compat mode. Check for these here too. */
898 leaq iret_label(%rip),%rbp
899 cmpq %rbp,RIP(%rsp)
900 je error_swapgs
901 movl %ebp,%ebp /* zero extend */
902 cmpq %rbp,RIP(%rsp)
903 je error_swapgs
904 cmpq $gs_change,RIP(%rsp)
905 je error_swapgs
906 jmp error_sti
Prasanna S.Pd28c4392006-09-26 10:52:34 +0200907KPROBE_END(error_entry)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908
909 /* Reload gs selector with exception handling */
910 /* edi: new selector */
911ENTRY(load_gs_index)
Jan Beulich7effaa82005-09-12 18:49:24 +0200912 CFI_STARTPROC
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 pushf
Jan Beulich7effaa82005-09-12 18:49:24 +0200914 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915 cli
916 swapgs
917gs_change:
918 movl %edi,%gs
9192: mfence /* workaround */
920 swapgs
921 popf
Jan Beulich7effaa82005-09-12 18:49:24 +0200922 CFI_ADJUST_CFA_OFFSET -8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923 ret
Jan Beulich7effaa82005-09-12 18:49:24 +0200924 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +0200925ENDPROC(load_gs_index)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926
927 .section __ex_table,"a"
928 .align 8
929 .quad gs_change,bad_gs
930 .previous
931 .section .fixup,"ax"
932 /* running with kernelgs */
933bad_gs:
934 swapgs /* switch back to user gs */
935 xorl %eax,%eax
936 movl %eax,%gs
937 jmp 2b
938 .previous
939
940/*
941 * Create a kernel thread.
942 *
943 * C extern interface:
944 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
945 *
946 * asm input arguments:
947 * rdi: fn, rsi: arg, rdx: flags
948 */
949ENTRY(kernel_thread)
950 CFI_STARTPROC
951 FAKE_STACK_FRAME $child_rip
952 SAVE_ALL
953
954 # rdi: flags, rsi: usp, rdx: will be &pt_regs
955 movq %rdx,%rdi
956 orq kernel_thread_flags(%rip),%rdi
957 movq $-1, %rsi
958 movq %rsp, %rdx
959
960 xorl %r8d,%r8d
961 xorl %r9d,%r9d
962
963 # clone now
964 call do_fork
965 movq %rax,RAX(%rsp)
966 xorl %edi,%edi
967
968 /*
969 * It isn't worth to check for reschedule here,
970 * so internally to the x86_64 port you can rely on kernel_thread()
971 * not to reschedule the child before returning, this avoids the need
972 * of hacks for example to fork off the per-CPU idle tasks.
973 * [Hopefully no generic code relies on the reschedule -AK]
974 */
975 RESTORE_ALL
976 UNFAKE_STACK_FRAME
977 ret
978 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +0200979ENDPROC(kernel_thread)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980
981child_rip:
Andi Kleenc05991e2006-08-30 19:37:08 +0200982 pushq $0 # fake return address
983 CFI_STARTPROC
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984 /*
985 * Here we are in the child and the registers are set as they were
986 * at kernel_thread() invocation in the parent.
987 */
988 movq %rdi, %rax
989 movq %rsi, %rdi
990 call *%rax
991 # exit
Andi Kleen3829ee62005-07-28 21:15:48 -0700992 xorl %edi, %edi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 call do_exit
Andi Kleenc05991e2006-08-30 19:37:08 +0200994 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +0200995ENDPROC(child_rip)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996
997/*
998 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
999 *
1000 * C extern interface:
1001 * extern long execve(char *name, char **argv, char **envp)
1002 *
1003 * asm input arguments:
1004 * rdi: name, rsi: argv, rdx: envp
1005 *
1006 * We want to fallback into:
1007 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
1008 *
1009 * do_sys_execve asm fallback arguments:
1010 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
1011 */
Arnd Bergmann3db03b42006-10-02 02:18:31 -07001012ENTRY(kernel_execve)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013 CFI_STARTPROC
1014 FAKE_STACK_FRAME $0
1015 SAVE_ALL
1016 call sys_execve
1017 movq %rax, RAX(%rsp)
1018 RESTORE_REST
1019 testq %rax,%rax
1020 je int_ret_from_sys_call
1021 RESTORE_ARGS
1022 UNFAKE_STACK_FRAME
1023 ret
1024 CFI_ENDPROC
Arnd Bergmann3db03b42006-10-02 02:18:31 -07001025ENDPROC(kernel_execve)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -07001027KPROBE_ENTRY(page_fault)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028 errorentry do_page_fault
Prasanna S.Pd28c4392006-09-26 10:52:34 +02001029KPROBE_END(page_fault)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030
1031ENTRY(coprocessor_error)
1032 zeroentry do_coprocessor_error
Jan Beulich4b787e02006-06-26 13:56:55 +02001033END(coprocessor_error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034
1035ENTRY(simd_coprocessor_error)
1036 zeroentry do_simd_coprocessor_error
Jan Beulich4b787e02006-06-26 13:56:55 +02001037END(simd_coprocessor_error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038
1039ENTRY(device_not_available)
1040 zeroentry math_state_restore
Jan Beulich4b787e02006-06-26 13:56:55 +02001041END(device_not_available)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042
1043 /* runs on exception stack */
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -07001044KPROBE_ENTRY(debug)
Jan Beulich7effaa82005-09-12 18:49:24 +02001045 INTR_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046 pushq $0
1047 CFI_ADJUST_CFA_OFFSET 8
Andi Kleen5f8efbb2006-01-16 01:56:39 +01001048 paranoidentry do_debug, DEBUG_STACK
Ingo Molnar2601e642006-07-03 00:24:45 -07001049 paranoidexit
Prasanna S.Pd28c4392006-09-26 10:52:34 +02001050KPROBE_END(debug)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051
1052 /* runs on exception stack */
Andi Kleeneddb6fb2006-02-03 21:50:41 +01001053KPROBE_ENTRY(nmi)
Jan Beulich7effaa82005-09-12 18:49:24 +02001054 INTR_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -07001055 pushq $-1
Jan Beulich7effaa82005-09-12 18:49:24 +02001056 CFI_ADJUST_CFA_OFFSET 8
Ingo Molnar2601e642006-07-03 00:24:45 -07001057 paranoidentry do_nmi, 0, 0
1058#ifdef CONFIG_TRACE_IRQFLAGS
1059 paranoidexit 0
1060#else
1061 jmp paranoid_exit1
1062 CFI_ENDPROC
1063#endif
Prasanna S.Pd28c4392006-09-26 10:52:34 +02001064KPROBE_END(nmi)
Andi Kleen6fefb0d2005-04-16 15:25:03 -07001065
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -07001066KPROBE_ENTRY(int3)
Jan Beulichb556b352006-01-11 22:43:00 +01001067 INTR_FRAME
1068 pushq $0
1069 CFI_ADJUST_CFA_OFFSET 8
Andi Kleen5f8efbb2006-01-16 01:56:39 +01001070 paranoidentry do_int3, DEBUG_STACK
Ingo Molnar2601e642006-07-03 00:24:45 -07001071 jmp paranoid_exit1
Jan Beulichb556b352006-01-11 22:43:00 +01001072 CFI_ENDPROC
Prasanna S.Pd28c4392006-09-26 10:52:34 +02001073KPROBE_END(int3)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074
1075ENTRY(overflow)
1076 zeroentry do_overflow
Jan Beulich4b787e02006-06-26 13:56:55 +02001077END(overflow)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078
1079ENTRY(bounds)
1080 zeroentry do_bounds
Jan Beulich4b787e02006-06-26 13:56:55 +02001081END(bounds)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082
1083ENTRY(invalid_op)
1084 zeroentry do_invalid_op
Jan Beulich4b787e02006-06-26 13:56:55 +02001085END(invalid_op)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001086
1087ENTRY(coprocessor_segment_overrun)
1088 zeroentry do_coprocessor_segment_overrun
Jan Beulich4b787e02006-06-26 13:56:55 +02001089END(coprocessor_segment_overrun)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090
1091ENTRY(reserved)
1092 zeroentry do_reserved
Jan Beulich4b787e02006-06-26 13:56:55 +02001093END(reserved)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094
1095 /* runs on exception stack */
1096ENTRY(double_fault)
Jan Beulich7effaa82005-09-12 18:49:24 +02001097 XCPT_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098 paranoidentry do_double_fault
Ingo Molnar2601e642006-07-03 00:24:45 -07001099 jmp paranoid_exit1
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +02001101END(double_fault)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102
1103ENTRY(invalid_TSS)
1104 errorentry do_invalid_TSS
Jan Beulich4b787e02006-06-26 13:56:55 +02001105END(invalid_TSS)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106
1107ENTRY(segment_not_present)
1108 errorentry do_segment_not_present
Jan Beulich4b787e02006-06-26 13:56:55 +02001109END(segment_not_present)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
1111 /* runs on exception stack */
1112ENTRY(stack_segment)
Jan Beulich7effaa82005-09-12 18:49:24 +02001113 XCPT_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -07001114 paranoidentry do_stack_segment
Ingo Molnar2601e642006-07-03 00:24:45 -07001115 jmp paranoid_exit1
Linus Torvalds1da177e2005-04-16 15:20:36 -07001116 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +02001117END(stack_segment)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -07001119KPROBE_ENTRY(general_protection)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120 errorentry do_general_protection
Prasanna S.Pd28c4392006-09-26 10:52:34 +02001121KPROBE_END(general_protection)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122
1123ENTRY(alignment_check)
1124 errorentry do_alignment_check
Jan Beulich4b787e02006-06-26 13:56:55 +02001125END(alignment_check)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126
1127ENTRY(divide_error)
1128 zeroentry do_divide_error
Jan Beulich4b787e02006-06-26 13:56:55 +02001129END(divide_error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001130
1131ENTRY(spurious_interrupt_bug)
1132 zeroentry do_spurious_interrupt_bug
Jan Beulich4b787e02006-06-26 13:56:55 +02001133END(spurious_interrupt_bug)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134
1135#ifdef CONFIG_X86_MCE
1136 /* runs on exception stack */
1137ENTRY(machine_check)
Jan Beulich7effaa82005-09-12 18:49:24 +02001138 INTR_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139 pushq $0
1140 CFI_ADJUST_CFA_OFFSET 8
1141 paranoidentry do_machine_check
Ingo Molnar2601e642006-07-03 00:24:45 -07001142 jmp paranoid_exit1
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +02001144END(machine_check)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145#endif
1146
Andi Kleen26995002006-08-02 22:37:28 +02001147/* Call softirq on interrupt stack. Interrupts are off. */
Andi Kleened6b6762005-07-28 21:15:49 -07001148ENTRY(call_softirq)
Jan Beulich7effaa82005-09-12 18:49:24 +02001149 CFI_STARTPROC
Andi Kleen26995002006-08-02 22:37:28 +02001150 push %rbp
1151 CFI_ADJUST_CFA_OFFSET 8
1152 CFI_REL_OFFSET rbp,0
1153 mov %rsp,%rbp
1154 CFI_DEF_CFA_REGISTER rbp
Andi Kleened6b6762005-07-28 21:15:49 -07001155 incl %gs:pda_irqcount
Andi Kleen26995002006-08-02 22:37:28 +02001156 cmove %gs:pda_irqstackptr,%rsp
1157 push %rbp # backlink for old unwinder
Andi Kleened6b6762005-07-28 21:15:49 -07001158 call __do_softirq
Andi Kleen26995002006-08-02 22:37:28 +02001159 leaveq
Jan Beulich7effaa82005-09-12 18:49:24 +02001160 CFI_DEF_CFA_REGISTER rsp
Andi Kleen26995002006-08-02 22:37:28 +02001161 CFI_ADJUST_CFA_OFFSET -8
Andi Kleened6b6762005-07-28 21:15:49 -07001162 decl %gs:pda_irqcount
Andi Kleened6b6762005-07-28 21:15:49 -07001163 ret
Jan Beulich7effaa82005-09-12 18:49:24 +02001164 CFI_ENDPROC
Jan Beulich4b787e02006-06-26 13:56:55 +02001165ENDPROC(call_softirq)