blob: 45e0ab8a1fcacbf0aba21dba719eeae5fcbad909 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86_64/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7 *
8 * $Id$
9 */
10
11/*
12 * entry.S contains the system-call and fault low-level handling routines.
13 *
14 * NOTE: This code handles signal-recognition, which happens every time
15 * after an interrupt and after each system call.
16 *
17 * Normal syscalls and interrupts don't save a full stack frame, this is
18 * only done for syscall tracing, signals or fork/exec et.al.
19 *
20 * A note on terminology:
21 * - top of stack: Architecture defined interrupt frame from SS to RIP
22 * at the top of the kernel process stack.
23 * - partial stack frame: partially saved registers upto R11.
24 * - full stack frame: Like partial stack frame, but all register saved.
25 *
26 * TODO:
27 * - schedule it carefully for the final hardware.
28 */
29
30#define ASSEMBLY 1
31#include <linux/config.h>
32#include <linux/linkage.h>
33#include <asm/segment.h>
34#include <asm/smp.h>
35#include <asm/cache.h>
36#include <asm/errno.h>
37#include <asm/dwarf2.h>
38#include <asm/calling.h>
Sam Ravnborge2d5df92005-09-09 21:28:48 +020039#include <asm/asm-offsets.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#include <asm/msr.h>
41#include <asm/unistd.h>
42#include <asm/thread_info.h>
43#include <asm/hw_irq.h>
44
45 .code64
46
Andi Kleendc37db42005-04-16 15:25:05 -070047#ifndef CONFIG_PREEMPT
Linus Torvalds1da177e2005-04-16 15:20:36 -070048#define retint_kernel retint_restore_args
49#endif
50
51/*
52 * C code is not supposed to know about undefined top of stack. Every time
53 * a C function with an pt_regs argument is called from the SYSCALL based
54 * fast path FIXUP_TOP_OF_STACK is needed.
55 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
56 * manipulation.
57 */
58
59 /* %rsp:at FRAMEEND */
60 .macro FIXUP_TOP_OF_STACK tmp
61 movq %gs:pda_oldrsp,\tmp
62 movq \tmp,RSP(%rsp)
63 movq $__USER_DS,SS(%rsp)
64 movq $__USER_CS,CS(%rsp)
65 movq $-1,RCX(%rsp)
66 movq R11(%rsp),\tmp /* get eflags */
67 movq \tmp,EFLAGS(%rsp)
68 .endm
69
70 .macro RESTORE_TOP_OF_STACK tmp,offset=0
71 movq RSP-\offset(%rsp),\tmp
72 movq \tmp,%gs:pda_oldrsp
73 movq EFLAGS-\offset(%rsp),\tmp
74 movq \tmp,R11-\offset(%rsp)
75 .endm
76
77 .macro FAKE_STACK_FRAME child_rip
78 /* push in order ss, rsp, eflags, cs, rip */
Andi Kleen3829ee62005-07-28 21:15:48 -070079 xorl %eax, %eax
Linus Torvalds1da177e2005-04-16 15:20:36 -070080 pushq %rax /* ss */
81 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich7effaa82005-09-12 18:49:24 +020082 /*CFI_REL_OFFSET ss,0*/
Linus Torvalds1da177e2005-04-16 15:20:36 -070083 pushq %rax /* rsp */
84 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich7effaa82005-09-12 18:49:24 +020085 CFI_REL_OFFSET rsp,0
Linus Torvalds1da177e2005-04-16 15:20:36 -070086 pushq $(1<<9) /* eflags - interrupts on */
87 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich7effaa82005-09-12 18:49:24 +020088 /*CFI_REL_OFFSET rflags,0*/
Linus Torvalds1da177e2005-04-16 15:20:36 -070089 pushq $__KERNEL_CS /* cs */
90 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich7effaa82005-09-12 18:49:24 +020091 /*CFI_REL_OFFSET cs,0*/
Linus Torvalds1da177e2005-04-16 15:20:36 -070092 pushq \child_rip /* rip */
93 CFI_ADJUST_CFA_OFFSET 8
Jan Beulich7effaa82005-09-12 18:49:24 +020094 CFI_REL_OFFSET rip,0
Linus Torvalds1da177e2005-04-16 15:20:36 -070095 pushq %rax /* orig rax */
96 CFI_ADJUST_CFA_OFFSET 8
97 .endm
98
99 .macro UNFAKE_STACK_FRAME
100 addq $8*6, %rsp
101 CFI_ADJUST_CFA_OFFSET -(6*8)
102 .endm
103
Jan Beulich7effaa82005-09-12 18:49:24 +0200104 .macro CFI_DEFAULT_STACK start=1
105 .if \start
106 CFI_STARTPROC simple
107 CFI_DEF_CFA rsp,SS+8
108 .else
109 CFI_DEF_CFA_OFFSET SS+8
110 .endif
111 CFI_REL_OFFSET r15,R15
112 CFI_REL_OFFSET r14,R14
113 CFI_REL_OFFSET r13,R13
114 CFI_REL_OFFSET r12,R12
115 CFI_REL_OFFSET rbp,RBP
116 CFI_REL_OFFSET rbx,RBX
117 CFI_REL_OFFSET r11,R11
118 CFI_REL_OFFSET r10,R10
119 CFI_REL_OFFSET r9,R9
120 CFI_REL_OFFSET r8,R8
121 CFI_REL_OFFSET rax,RAX
122 CFI_REL_OFFSET rcx,RCX
123 CFI_REL_OFFSET rdx,RDX
124 CFI_REL_OFFSET rsi,RSI
125 CFI_REL_OFFSET rdi,RDI
126 CFI_REL_OFFSET rip,RIP
127 /*CFI_REL_OFFSET cs,CS*/
128 /*CFI_REL_OFFSET rflags,EFLAGS*/
129 CFI_REL_OFFSET rsp,RSP
130 /*CFI_REL_OFFSET ss,SS*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131 .endm
132/*
133 * A newly forked process directly context switches into this.
134 */
135/* rdi: prev */
136ENTRY(ret_from_fork)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 CFI_DEFAULT_STACK
138 call schedule_tail
139 GET_THREAD_INFO(%rcx)
140 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
141 jnz rff_trace
142rff_action:
143 RESTORE_REST
144 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
145 je int_ret_from_sys_call
146 testl $_TIF_IA32,threadinfo_flags(%rcx)
147 jnz int_ret_from_sys_call
148 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
149 jmp ret_from_sys_call
150rff_trace:
151 movq %rsp,%rdi
152 call syscall_trace_leave
153 GET_THREAD_INFO(%rcx)
154 jmp rff_action
155 CFI_ENDPROC
156
157/*
158 * System call entry. Upto 6 arguments in registers are supported.
159 *
160 * SYSCALL does not save anything on the stack and does not change the
161 * stack pointer.
162 */
163
164/*
165 * Register setup:
166 * rax system call number
167 * rdi arg0
168 * rcx return address for syscall/sysret, C arg3
169 * rsi arg1
170 * rdx arg2
171 * r10 arg3 (--> moved to rcx for C)
172 * r8 arg4
173 * r9 arg5
174 * r11 eflags for syscall/sysret, temporary for C
175 * r12-r15,rbp,rbx saved by C code, not touched.
176 *
177 * Interrupts are off on entry.
178 * Only called from user space.
179 *
180 * XXX if we had a free scratch register we could save the RSP into the stack frame
181 * and report it properly in ps. Unfortunately we haven't.
182 */
183
184ENTRY(system_call)
Jan Beulich7effaa82005-09-12 18:49:24 +0200185 CFI_STARTPROC simple
186 CFI_DEF_CFA rsp,0
187 CFI_REGISTER rip,rcx
188 /*CFI_REGISTER rflags,r11*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189 swapgs
190 movq %rsp,%gs:pda_oldrsp
191 movq %gs:pda_kernelstack,%rsp
192 sti
193 SAVE_ARGS 8,1
194 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
Jan Beulich7effaa82005-09-12 18:49:24 +0200195 movq %rcx,RIP-ARGOFFSET(%rsp)
196 CFI_REL_OFFSET rip,RIP-ARGOFFSET
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 GET_THREAD_INFO(%rcx)
198 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
Jan Beulich7effaa82005-09-12 18:49:24 +0200199 CFI_REMEMBER_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 jnz tracesys
201 cmpq $__NR_syscall_max,%rax
202 ja badsys
203 movq %r10,%rcx
204 call *sys_call_table(,%rax,8) # XXX: rip relative
205 movq %rax,RAX-ARGOFFSET(%rsp)
206/*
207 * Syscall return path ending with SYSRET (fast path)
208 * Has incomplete stack frame and undefined top of stack.
209 */
210 .globl ret_from_sys_call
211ret_from_sys_call:
Andi Kleen11b854b2005-04-16 15:25:02 -0700212 movl $_TIF_ALLWORK_MASK,%edi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 /* edi: flagmask */
214sysret_check:
215 GET_THREAD_INFO(%rcx)
216 cli
217 movl threadinfo_flags(%rcx),%edx
218 andl %edi,%edx
Jan Beulich7effaa82005-09-12 18:49:24 +0200219 CFI_REMEMBER_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 jnz sysret_careful
221 movq RIP-ARGOFFSET(%rsp),%rcx
Jan Beulich7effaa82005-09-12 18:49:24 +0200222 CFI_REGISTER rip,rcx
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 RESTORE_ARGS 0,-ARG_SKIP,1
Jan Beulich7effaa82005-09-12 18:49:24 +0200224 /*CFI_REGISTER rflags,r11*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 movq %gs:pda_oldrsp,%rsp
226 swapgs
227 sysretq
228
229 /* Handle reschedules */
230 /* edx: work, edi: workmask */
231sysret_careful:
Jan Beulich7effaa82005-09-12 18:49:24 +0200232 CFI_RESTORE_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233 bt $TIF_NEED_RESCHED,%edx
234 jnc sysret_signal
235 sti
236 pushq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200237 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 call schedule
239 popq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200240 CFI_ADJUST_CFA_OFFSET -8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241 jmp sysret_check
242
243 /* Handle a signal */
244sysret_signal:
245 sti
Andi Kleen10ffdbb2005-05-16 21:53:19 -0700246 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
247 jz 1f
248
249 /* Really a signal */
250 /* edx: work flags (arg3) */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 leaq do_notify_resume(%rip),%rax
252 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
253 xorl %esi,%esi # oldset -> arg2
254 call ptregscall_common
Andi Kleen10ffdbb2005-05-16 21:53:19 -07002551: movl $_TIF_NEED_RESCHED,%edi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 jmp sysret_check
257
Jan Beulich7effaa82005-09-12 18:49:24 +0200258badsys:
259 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
260 jmp ret_from_sys_call
261
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 /* Do syscall tracing */
263tracesys:
Jan Beulich7effaa82005-09-12 18:49:24 +0200264 CFI_RESTORE_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265 SAVE_REST
266 movq $-ENOSYS,RAX(%rsp)
267 FIXUP_TOP_OF_STACK %rdi
268 movq %rsp,%rdi
269 call syscall_trace_enter
270 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
271 RESTORE_REST
272 cmpq $__NR_syscall_max,%rax
273 ja 1f
274 movq %r10,%rcx /* fixup for C */
275 call *sys_call_table(,%rax,8)
276 movq %rax,RAX-ARGOFFSET(%rsp)
2771: SAVE_REST
278 movq %rsp,%rdi
279 call syscall_trace_leave
280 RESTORE_TOP_OF_STACK %rbx
281 RESTORE_REST
282 jmp ret_from_sys_call
Jan Beulich7effaa82005-09-12 18:49:24 +0200283 CFI_ENDPROC
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285/*
286 * Syscall return path ending with IRET.
287 * Has correct top of stack, but partial stack frame.
288 */
Jan Beulich7effaa82005-09-12 18:49:24 +0200289ENTRY(int_ret_from_sys_call)
290 CFI_STARTPROC simple
291 CFI_DEF_CFA rsp,SS+8-ARGOFFSET
292 /*CFI_REL_OFFSET ss,SS-ARGOFFSET*/
293 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
294 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
295 /*CFI_REL_OFFSET cs,CS-ARGOFFSET*/
296 CFI_REL_OFFSET rip,RIP-ARGOFFSET
297 CFI_REL_OFFSET rdx,RDX-ARGOFFSET
298 CFI_REL_OFFSET rcx,RCX-ARGOFFSET
299 CFI_REL_OFFSET rax,RAX-ARGOFFSET
300 CFI_REL_OFFSET rdi,RDI-ARGOFFSET
301 CFI_REL_OFFSET rsi,RSI-ARGOFFSET
302 CFI_REL_OFFSET r8,R8-ARGOFFSET
303 CFI_REL_OFFSET r9,R9-ARGOFFSET
304 CFI_REL_OFFSET r10,R10-ARGOFFSET
305 CFI_REL_OFFSET r11,R11-ARGOFFSET
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 cli
307 testl $3,CS-ARGOFFSET(%rsp)
308 je retint_restore_args
309 movl $_TIF_ALLWORK_MASK,%edi
310 /* edi: mask to check */
311int_with_check:
312 GET_THREAD_INFO(%rcx)
313 movl threadinfo_flags(%rcx),%edx
314 andl %edi,%edx
315 jnz int_careful
316 jmp retint_swapgs
317
318 /* Either reschedule or signal or syscall exit tracking needed. */
319 /* First do a reschedule test. */
320 /* edx: work, edi: workmask */
321int_careful:
322 bt $TIF_NEED_RESCHED,%edx
323 jnc int_very_careful
324 sti
325 pushq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200326 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327 call schedule
328 popq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200329 CFI_ADJUST_CFA_OFFSET -8
Andi Kleencdd219c2005-04-16 15:25:04 -0700330 cli
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 jmp int_with_check
332
333 /* handle signals and tracing -- both require a full stack frame */
334int_very_careful:
335 sti
336 SAVE_REST
337 /* Check for syscall exit trace */
338 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
339 jz int_signal
340 pushq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200341 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342 leaq 8(%rsp),%rdi # &ptregs -> arg1
343 call syscall_trace_leave
344 popq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200345 CFI_ADJUST_CFA_OFFSET -8
Andi Kleen36c11042005-04-16 15:25:01 -0700346 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
Andi Kleenbe9e6872005-05-01 08:58:51 -0700347 cli
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 jmp int_restore_rest
349
350int_signal:
351 testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
352 jz 1f
353 movq %rsp,%rdi # &ptregs -> arg1
354 xorl %esi,%esi # oldset -> arg2
355 call do_notify_resume
3561: movl $_TIF_NEED_RESCHED,%edi
357int_restore_rest:
358 RESTORE_REST
Andi Kleenbe9e6872005-05-01 08:58:51 -0700359 cli
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 jmp int_with_check
361 CFI_ENDPROC
362
363/*
364 * Certain special system calls that need to save a complete full stack frame.
365 */
366
367 .macro PTREGSCALL label,func,arg
368 .globl \label
369\label:
370 leaq \func(%rip),%rax
371 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
372 jmp ptregscall_common
373 .endm
374
Jan Beulich7effaa82005-09-12 18:49:24 +0200375 CFI_STARTPROC
376
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 PTREGSCALL stub_clone, sys_clone, %r8
378 PTREGSCALL stub_fork, sys_fork, %rdi
379 PTREGSCALL stub_vfork, sys_vfork, %rdi
380 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
381 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
382 PTREGSCALL stub_iopl, sys_iopl, %rsi
383
384ENTRY(ptregscall_common)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385 popq %r11
Jan Beulich7effaa82005-09-12 18:49:24 +0200386 CFI_ADJUST_CFA_OFFSET -8
387 CFI_REGISTER rip, r11
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 SAVE_REST
389 movq %r11, %r15
Jan Beulich7effaa82005-09-12 18:49:24 +0200390 CFI_REGISTER rip, r15
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391 FIXUP_TOP_OF_STACK %r11
392 call *%rax
393 RESTORE_TOP_OF_STACK %r11
394 movq %r15, %r11
Jan Beulich7effaa82005-09-12 18:49:24 +0200395 CFI_REGISTER rip, r11
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 RESTORE_REST
397 pushq %r11
Jan Beulich7effaa82005-09-12 18:49:24 +0200398 CFI_ADJUST_CFA_OFFSET 8
399 CFI_REL_OFFSET rip, 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400 ret
401 CFI_ENDPROC
402
403ENTRY(stub_execve)
404 CFI_STARTPROC
405 popq %r11
Jan Beulich7effaa82005-09-12 18:49:24 +0200406 CFI_ADJUST_CFA_OFFSET -8
407 CFI_REGISTER rip, r11
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 SAVE_REST
409 movq %r11, %r15
Jan Beulich7effaa82005-09-12 18:49:24 +0200410 CFI_REGISTER rip, r15
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 FIXUP_TOP_OF_STACK %r11
412 call sys_execve
413 GET_THREAD_INFO(%rcx)
414 bt $TIF_IA32,threadinfo_flags(%rcx)
Jan Beulich7effaa82005-09-12 18:49:24 +0200415 CFI_REMEMBER_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 jc exec_32bit
417 RESTORE_TOP_OF_STACK %r11
418 movq %r15, %r11
Jan Beulich7effaa82005-09-12 18:49:24 +0200419 CFI_REGISTER rip, r11
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 RESTORE_REST
Jan Beulich7effaa82005-09-12 18:49:24 +0200421 pushq %r11
422 CFI_ADJUST_CFA_OFFSET 8
423 CFI_REL_OFFSET rip, 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 ret
425
426exec_32bit:
Jan Beulich7effaa82005-09-12 18:49:24 +0200427 CFI_RESTORE_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 movq %rax,RAX(%rsp)
429 RESTORE_REST
430 jmp int_ret_from_sys_call
431 CFI_ENDPROC
432
433/*
434 * sigreturn is special because it needs to restore all registers on return.
435 * This cannot be done with SYSRET, so use the IRET return path instead.
436 */
437ENTRY(stub_rt_sigreturn)
438 CFI_STARTPROC
Jan Beulich7effaa82005-09-12 18:49:24 +0200439 addq $8, %rsp
440 CFI_ADJUST_CFA_OFFSET -8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 SAVE_REST
442 movq %rsp,%rdi
443 FIXUP_TOP_OF_STACK %r11
444 call sys_rt_sigreturn
445 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
446 RESTORE_REST
447 jmp int_ret_from_sys_call
448 CFI_ENDPROC
449
Jan Beulich7effaa82005-09-12 18:49:24 +0200450/*
451 * initial frame state for interrupts and exceptions
452 */
453 .macro _frame ref
454 CFI_STARTPROC simple
455 CFI_DEF_CFA rsp,SS+8-\ref
456 /*CFI_REL_OFFSET ss,SS-\ref*/
457 CFI_REL_OFFSET rsp,RSP-\ref
458 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
459 /*CFI_REL_OFFSET cs,CS-\ref*/
460 CFI_REL_OFFSET rip,RIP-\ref
461 .endm
462
463/* initial frame state for interrupts (and exceptions without error code) */
464#define INTR_FRAME _frame RIP
465/* initial frame state for exceptions with error code (and interrupts with
466 vector already pushed) */
467#define XCPT_FRAME _frame ORIG_RAX
468
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469/*
470 * Interrupt entry/exit.
471 *
472 * Interrupt entry points save only callee clobbered registers in fast path.
473 *
474 * Entry runs with interrupts off.
475 */
476
477/* 0(%rsp): interrupt number */
478 .macro interrupt func
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479 cld
480#ifdef CONFIG_DEBUG_INFO
481 SAVE_ALL
482 movq %rsp,%rdi
483 /*
484 * Setup a stack frame pointer. This allows gdb to trace
485 * back to the original stack.
486 */
487 movq %rsp,%rbp
488 CFI_DEF_CFA_REGISTER rbp
489#else
490 SAVE_ARGS
491 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
492#endif
493 testl $3,CS(%rdi)
494 je 1f
495 swapgs
Andi Kleen3829ee62005-07-28 21:15:48 -07004961: incl %gs:pda_irqcount # RED-PEN should check preempt count
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 movq %gs:pda_irqstackptr,%rax
Jan Beulich7effaa82005-09-12 18:49:24 +0200498 cmoveq %rax,%rsp /*todo This needs CFI annotation! */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 pushq %rdi # save old stack
Jan Beulich7effaa82005-09-12 18:49:24 +0200500 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 call \func
502 .endm
503
504ENTRY(common_interrupt)
Jan Beulich7effaa82005-09-12 18:49:24 +0200505 XCPT_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 interrupt do_IRQ
507 /* 0(%rsp): oldrsp-ARGOFFSET */
Jan Beulich7effaa82005-09-12 18:49:24 +0200508ret_from_intr:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 popq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200510 CFI_ADJUST_CFA_OFFSET -8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 cli
Andi Kleen3829ee62005-07-28 21:15:48 -0700512 decl %gs:pda_irqcount
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513#ifdef CONFIG_DEBUG_INFO
514 movq RBP(%rdi),%rbp
Jan Beulich7effaa82005-09-12 18:49:24 +0200515 CFI_DEF_CFA_REGISTER rsp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516#endif
Jan Beulich7effaa82005-09-12 18:49:24 +0200517 leaq ARGOFFSET(%rdi),%rsp /*todo This needs CFI annotation! */
518exit_intr:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 GET_THREAD_INFO(%rcx)
520 testl $3,CS-ARGOFFSET(%rsp)
521 je retint_kernel
522
523 /* Interrupt came from user space */
524 /*
525 * Has a correct top of stack, but a partial stack frame
526 * %rcx: thread info. Interrupts off.
527 */
528retint_with_reschedule:
529 movl $_TIF_WORK_MASK,%edi
Jan Beulich7effaa82005-09-12 18:49:24 +0200530retint_check:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 movl threadinfo_flags(%rcx),%edx
532 andl %edi,%edx
Jan Beulich7effaa82005-09-12 18:49:24 +0200533 CFI_REMEMBER_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 jnz retint_careful
535retint_swapgs:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 swapgs
537retint_restore_args:
538 cli
539 RESTORE_ARGS 0,8,0
540iret_label:
541 iretq
542
543 .section __ex_table,"a"
544 .quad iret_label,bad_iret
545 .previous
546 .section .fixup,"ax"
547 /* force a signal here? this matches i386 behaviour */
548 /* running with kernel gs */
549bad_iret:
550 movq $-9999,%rdi /* better code? */
551 jmp do_exit
552 .previous
553
Jan Beulich7effaa82005-09-12 18:49:24 +0200554 /* edi: workmask, edx: work */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555retint_careful:
Jan Beulich7effaa82005-09-12 18:49:24 +0200556 CFI_RESTORE_STATE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 bt $TIF_NEED_RESCHED,%edx
558 jnc retint_signal
559 sti
560 pushq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200561 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 call schedule
563 popq %rdi
Jan Beulich7effaa82005-09-12 18:49:24 +0200564 CFI_ADJUST_CFA_OFFSET -8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 GET_THREAD_INFO(%rcx)
566 cli
567 jmp retint_check
568
569retint_signal:
Andi Kleen10ffdbb2005-05-16 21:53:19 -0700570 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
571 jz retint_swapgs
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 sti
573 SAVE_REST
574 movq $-1,ORIG_RAX(%rsp)
Andi Kleen3829ee62005-07-28 21:15:48 -0700575 xorl %esi,%esi # oldset
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 movq %rsp,%rdi # &pt_regs
577 call do_notify_resume
578 RESTORE_REST
579 cli
Andi Kleen10ffdbb2005-05-16 21:53:19 -0700580 movl $_TIF_NEED_RESCHED,%edi
Andi Kleenbe9e6872005-05-01 08:58:51 -0700581 GET_THREAD_INFO(%rcx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 jmp retint_check
583
584#ifdef CONFIG_PREEMPT
585 /* Returning to kernel space. Check if we need preemption */
586 /* rcx: threadinfo. interrupts off. */
587 .p2align
588retint_kernel:
589 cmpl $0,threadinfo_preempt_count(%rcx)
590 jnz retint_restore_args
591 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
592 jnc retint_restore_args
593 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
594 jnc retint_restore_args
595 call preempt_schedule_irq
596 jmp exit_intr
597#endif
598 CFI_ENDPROC
599
600/*
601 * APIC interrupts.
602 */
603 .macro apicinterrupt num,func
Jan Beulich7effaa82005-09-12 18:49:24 +0200604 INTR_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 pushq $\num-256
Jan Beulich7effaa82005-09-12 18:49:24 +0200606 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 interrupt \func
608 jmp ret_from_intr
609 CFI_ENDPROC
610 .endm
611
612ENTRY(thermal_interrupt)
613 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
614
Jacob Shin89b831e2005-11-05 17:25:53 +0100615ENTRY(threshold_interrupt)
616 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
617
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618#ifdef CONFIG_SMP
619ENTRY(reschedule_interrupt)
620 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
621
Andi Kleene5bc8b62005-09-12 18:49:24 +0200622 .macro INVALIDATE_ENTRY num
623ENTRY(invalidate_interrupt\num)
624 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
625 .endm
626
627 INVALIDATE_ENTRY 0
628 INVALIDATE_ENTRY 1
629 INVALIDATE_ENTRY 2
630 INVALIDATE_ENTRY 3
631 INVALIDATE_ENTRY 4
632 INVALIDATE_ENTRY 5
633 INVALIDATE_ENTRY 6
634 INVALIDATE_ENTRY 7
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635
636ENTRY(call_function_interrupt)
637 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
638#endif
639
640#ifdef CONFIG_X86_LOCAL_APIC
641ENTRY(apic_timer_interrupt)
642 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
643
644ENTRY(error_interrupt)
645 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
646
647ENTRY(spurious_interrupt)
648 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
649#endif
650
651/*
652 * Exception entry points.
653 */
654 .macro zeroentry sym
Jan Beulich7effaa82005-09-12 18:49:24 +0200655 INTR_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 pushq $0 /* push error code/oldrax */
Jan Beulich7effaa82005-09-12 18:49:24 +0200657 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658 pushq %rax /* push real oldrax to the rdi slot */
Jan Beulich7effaa82005-09-12 18:49:24 +0200659 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660 leaq \sym(%rip),%rax
661 jmp error_entry
Jan Beulich7effaa82005-09-12 18:49:24 +0200662 CFI_ENDPROC
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 .endm
664
665 .macro errorentry sym
Jan Beulich7effaa82005-09-12 18:49:24 +0200666 XCPT_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667 pushq %rax
Jan Beulich7effaa82005-09-12 18:49:24 +0200668 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669 leaq \sym(%rip),%rax
670 jmp error_entry
Jan Beulich7effaa82005-09-12 18:49:24 +0200671 CFI_ENDPROC
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672 .endm
673
674 /* error code is on the stack already */
675 /* handle NMI like exceptions that can happen everywhere */
Jan Beulichb556b352006-01-11 22:43:00 +0100676#ifndef DEBUG_IST
677# define DEBUG_IST 0
678#endif
679 .macro paranoidentry sym, ist=0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 SAVE_ALL
681 cld
682 movl $1,%ebx
683 movl $MSR_GS_BASE,%ecx
684 rdmsr
685 testl %edx,%edx
686 js 1f
687 swapgs
688 xorl %ebx,%ebx
Jan Beulichb556b352006-01-11 22:43:00 +01006891:
690 .if \ist
691 movq %gs:pda_data_offset, %rbp
692 .endif
693 movq %rsp,%rdi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 movq ORIG_RAX(%rsp),%rsi
695 movq $-1,ORIG_RAX(%rsp)
Jan Beulichb556b352006-01-11 22:43:00 +0100696 .if \ist
697 subq $EXCEPTION_STACK_SIZE, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
698 .endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 call \sym
Jan Beulichb556b352006-01-11 22:43:00 +0100700 .if \ist
701 addq $EXCEPTION_STACK_SIZE, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
702 .endif
Andi Kleen6fefb0d2005-04-16 15:25:03 -0700703 cli
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 .endm
705
706/*
707 * Exception entry point. This expects an error code/orig_rax on the stack
708 * and the exception handler in %rax.
709 */
710ENTRY(error_entry)
Jan Beulich7effaa82005-09-12 18:49:24 +0200711 _frame RDI
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712 /* rdi slot contains rax, oldrax contains error code */
713 cld
714 subq $14*8,%rsp
715 CFI_ADJUST_CFA_OFFSET (14*8)
716 movq %rsi,13*8(%rsp)
717 CFI_REL_OFFSET rsi,RSI
718 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
719 movq %rdx,12*8(%rsp)
720 CFI_REL_OFFSET rdx,RDX
721 movq %rcx,11*8(%rsp)
722 CFI_REL_OFFSET rcx,RCX
723 movq %rsi,10*8(%rsp) /* store rax */
724 CFI_REL_OFFSET rax,RAX
725 movq %r8, 9*8(%rsp)
726 CFI_REL_OFFSET r8,R8
727 movq %r9, 8*8(%rsp)
728 CFI_REL_OFFSET r9,R9
729 movq %r10,7*8(%rsp)
730 CFI_REL_OFFSET r10,R10
731 movq %r11,6*8(%rsp)
732 CFI_REL_OFFSET r11,R11
733 movq %rbx,5*8(%rsp)
734 CFI_REL_OFFSET rbx,RBX
735 movq %rbp,4*8(%rsp)
736 CFI_REL_OFFSET rbp,RBP
737 movq %r12,3*8(%rsp)
738 CFI_REL_OFFSET r12,R12
739 movq %r13,2*8(%rsp)
740 CFI_REL_OFFSET r13,R13
741 movq %r14,1*8(%rsp)
742 CFI_REL_OFFSET r14,R14
743 movq %r15,(%rsp)
744 CFI_REL_OFFSET r15,R15
745 xorl %ebx,%ebx
746 testl $3,CS(%rsp)
747 je error_kernelspace
748error_swapgs:
749 swapgs
750error_sti:
751 movq %rdi,RDI(%rsp)
752 movq %rsp,%rdi
753 movq ORIG_RAX(%rsp),%rsi /* get error code */
754 movq $-1,ORIG_RAX(%rsp)
755 call *%rax
756 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
757error_exit:
758 movl %ebx,%eax
759 RESTORE_REST
760 cli
761 GET_THREAD_INFO(%rcx)
762 testl %eax,%eax
763 jne retint_kernel
764 movl threadinfo_flags(%rcx),%edx
765 movl $_TIF_WORK_MASK,%edi
766 andl %edi,%edx
767 jnz retint_careful
768 swapgs
769 RESTORE_ARGS 0,8,0
Jan Beulich505cc4e2006-01-11 22:42:20 +0100770 jmp iret_label
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771 CFI_ENDPROC
772
773error_kernelspace:
774 incl %ebx
775 /* There are two places in the kernel that can potentially fault with
776 usergs. Handle them here. The exception handlers after
777 iret run with kernel gs again, so don't set the user space flag.
778 B stepping K8s sometimes report an truncated RIP for IRET
779 exceptions returning to compat mode. Check for these here too. */
780 leaq iret_label(%rip),%rbp
781 cmpq %rbp,RIP(%rsp)
782 je error_swapgs
783 movl %ebp,%ebp /* zero extend */
784 cmpq %rbp,RIP(%rsp)
785 je error_swapgs
786 cmpq $gs_change,RIP(%rsp)
787 je error_swapgs
788 jmp error_sti
789
790 /* Reload gs selector with exception handling */
791 /* edi: new selector */
792ENTRY(load_gs_index)
Jan Beulich7effaa82005-09-12 18:49:24 +0200793 CFI_STARTPROC
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794 pushf
Jan Beulich7effaa82005-09-12 18:49:24 +0200795 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796 cli
797 swapgs
798gs_change:
799 movl %edi,%gs
8002: mfence /* workaround */
801 swapgs
802 popf
Jan Beulich7effaa82005-09-12 18:49:24 +0200803 CFI_ADJUST_CFA_OFFSET -8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804 ret
Jan Beulich7effaa82005-09-12 18:49:24 +0200805 CFI_ENDPROC
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806
807 .section __ex_table,"a"
808 .align 8
809 .quad gs_change,bad_gs
810 .previous
811 .section .fixup,"ax"
812 /* running with kernelgs */
813bad_gs:
814 swapgs /* switch back to user gs */
815 xorl %eax,%eax
816 movl %eax,%gs
817 jmp 2b
818 .previous
819
820/*
821 * Create a kernel thread.
822 *
823 * C extern interface:
824 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
825 *
826 * asm input arguments:
827 * rdi: fn, rsi: arg, rdx: flags
828 */
829ENTRY(kernel_thread)
830 CFI_STARTPROC
831 FAKE_STACK_FRAME $child_rip
832 SAVE_ALL
833
834 # rdi: flags, rsi: usp, rdx: will be &pt_regs
835 movq %rdx,%rdi
836 orq kernel_thread_flags(%rip),%rdi
837 movq $-1, %rsi
838 movq %rsp, %rdx
839
840 xorl %r8d,%r8d
841 xorl %r9d,%r9d
842
843 # clone now
844 call do_fork
845 movq %rax,RAX(%rsp)
846 xorl %edi,%edi
847
848 /*
849 * It isn't worth to check for reschedule here,
850 * so internally to the x86_64 port you can rely on kernel_thread()
851 * not to reschedule the child before returning, this avoids the need
852 * of hacks for example to fork off the per-CPU idle tasks.
853 * [Hopefully no generic code relies on the reschedule -AK]
854 */
855 RESTORE_ALL
856 UNFAKE_STACK_FRAME
857 ret
858 CFI_ENDPROC
859
860
861child_rip:
862 /*
863 * Here we are in the child and the registers are set as they were
864 * at kernel_thread() invocation in the parent.
865 */
866 movq %rdi, %rax
867 movq %rsi, %rdi
868 call *%rax
869 # exit
Andi Kleen3829ee62005-07-28 21:15:48 -0700870 xorl %edi, %edi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871 call do_exit
872
873/*
874 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
875 *
876 * C extern interface:
877 * extern long execve(char *name, char **argv, char **envp)
878 *
879 * asm input arguments:
880 * rdi: name, rsi: argv, rdx: envp
881 *
882 * We want to fallback into:
883 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
884 *
885 * do_sys_execve asm fallback arguments:
886 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
887 */
888ENTRY(execve)
889 CFI_STARTPROC
890 FAKE_STACK_FRAME $0
891 SAVE_ALL
892 call sys_execve
893 movq %rax, RAX(%rsp)
894 RESTORE_REST
895 testq %rax,%rax
896 je int_ret_from_sys_call
897 RESTORE_ARGS
898 UNFAKE_STACK_FRAME
899 ret
900 CFI_ENDPROC
901
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700902KPROBE_ENTRY(page_fault)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 errorentry do_page_fault
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700904 .previous .text
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905
906ENTRY(coprocessor_error)
907 zeroentry do_coprocessor_error
908
909ENTRY(simd_coprocessor_error)
910 zeroentry do_simd_coprocessor_error
911
912ENTRY(device_not_available)
913 zeroentry math_state_restore
914
915 /* runs on exception stack */
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700916KPROBE_ENTRY(debug)
Jan Beulich7effaa82005-09-12 18:49:24 +0200917 INTR_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918 pushq $0
919 CFI_ADJUST_CFA_OFFSET 8
Jan Beulichb556b352006-01-11 22:43:00 +0100920 paranoidentry do_debug, DEBUG_IST
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921 jmp paranoid_exit
922 CFI_ENDPROC
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700923 .previous .text
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924
925 /* runs on exception stack */
926ENTRY(nmi)
Jan Beulich7effaa82005-09-12 18:49:24 +0200927 INTR_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 pushq $-1
Jan Beulich7effaa82005-09-12 18:49:24 +0200929 CFI_ADJUST_CFA_OFFSET 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930 paranoidentry do_nmi
Andi Kleen6fefb0d2005-04-16 15:25:03 -0700931 /*
932 * "Paranoid" exit path from exception stack.
933 * Paranoid because this is used by NMIs and cannot take
934 * any kernel state for granted.
935 * We don't do kernel preemption checks here, because only
936 * NMI should be common and it does not enable IRQs and
937 * cannot get reschedule ticks.
938 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939 /* ebx: no swapgs flag */
940paranoid_exit:
941 testl %ebx,%ebx /* swapgs needed? */
942 jnz paranoid_restore
Andi Kleen6fefb0d2005-04-16 15:25:03 -0700943 testl $3,CS(%rsp)
944 jnz paranoid_userspace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945paranoid_swapgs:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 swapgs
947paranoid_restore:
948 RESTORE_ALL 8
949 iretq
950paranoid_userspace:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951 GET_THREAD_INFO(%rcx)
Andi Kleen6fefb0d2005-04-16 15:25:03 -0700952 movl threadinfo_flags(%rcx),%ebx
953 andl $_TIF_WORK_MASK,%ebx
Andi Kleen11b854b2005-04-16 15:25:02 -0700954 jz paranoid_swapgs
Andi Kleen6fefb0d2005-04-16 15:25:03 -0700955 movq %rsp,%rdi /* &pt_regs */
956 call sync_regs
957 movq %rax,%rsp /* switch stack for scheduling */
958 testl $_TIF_NEED_RESCHED,%ebx
959 jnz paranoid_schedule
960 movl %ebx,%edx /* arg3: thread flags */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961 sti
Andi Kleen6fefb0d2005-04-16 15:25:03 -0700962 xorl %esi,%esi /* arg2: oldset */
963 movq %rsp,%rdi /* arg1: &pt_regs */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 call do_notify_resume
Andi Kleen6fefb0d2005-04-16 15:25:03 -0700965 cli
966 jmp paranoid_userspace
967paranoid_schedule:
Andi Kleen11b854b2005-04-16 15:25:02 -0700968 sti
969 call schedule
Andi Kleen6fefb0d2005-04-16 15:25:03 -0700970 cli
971 jmp paranoid_userspace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 CFI_ENDPROC
Andi Kleen6fefb0d2005-04-16 15:25:03 -0700973
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700974KPROBE_ENTRY(int3)
Jan Beulichb556b352006-01-11 22:43:00 +0100975 INTR_FRAME
976 pushq $0
977 CFI_ADJUST_CFA_OFFSET 8
978 paranoidentry do_int3, DEBUG_IST
979 jmp paranoid_exit
980 CFI_ENDPROC
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700981 .previous .text
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982
983ENTRY(overflow)
984 zeroentry do_overflow
985
986ENTRY(bounds)
987 zeroentry do_bounds
988
989ENTRY(invalid_op)
990 zeroentry do_invalid_op
991
992ENTRY(coprocessor_segment_overrun)
993 zeroentry do_coprocessor_segment_overrun
994
995ENTRY(reserved)
996 zeroentry do_reserved
997
998 /* runs on exception stack */
999ENTRY(double_fault)
Jan Beulich7effaa82005-09-12 18:49:24 +02001000 XCPT_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 paranoidentry do_double_fault
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002 jmp paranoid_exit
1003 CFI_ENDPROC
1004
1005ENTRY(invalid_TSS)
1006 errorentry do_invalid_TSS
1007
1008ENTRY(segment_not_present)
1009 errorentry do_segment_not_present
1010
1011 /* runs on exception stack */
1012ENTRY(stack_segment)
Jan Beulich7effaa82005-09-12 18:49:24 +02001013 XCPT_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014 paranoidentry do_stack_segment
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015 jmp paranoid_exit
1016 CFI_ENDPROC
1017
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -07001018KPROBE_ENTRY(general_protection)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019 errorentry do_general_protection
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -07001020 .previous .text
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021
1022ENTRY(alignment_check)
1023 errorentry do_alignment_check
1024
1025ENTRY(divide_error)
1026 zeroentry do_divide_error
1027
1028ENTRY(spurious_interrupt_bug)
1029 zeroentry do_spurious_interrupt_bug
1030
1031#ifdef CONFIG_X86_MCE
1032 /* runs on exception stack */
1033ENTRY(machine_check)
Jan Beulich7effaa82005-09-12 18:49:24 +02001034 INTR_FRAME
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035 pushq $0
1036 CFI_ADJUST_CFA_OFFSET 8
1037 paranoidentry do_machine_check
1038 jmp paranoid_exit
1039 CFI_ENDPROC
1040#endif
1041
1042ENTRY(call_debug)
1043 zeroentry do_call_debug
1044
Andi Kleened6b6762005-07-28 21:15:49 -07001045ENTRY(call_softirq)
Jan Beulich7effaa82005-09-12 18:49:24 +02001046 CFI_STARTPROC
Andi Kleened6b6762005-07-28 21:15:49 -07001047 movq %gs:pda_irqstackptr,%rax
Jan Beulichbd9cb642006-01-11 22:43:21 +01001048 movq %rsp,%rdx
1049 CFI_DEF_CFA_REGISTER rdx
Andi Kleened6b6762005-07-28 21:15:49 -07001050 incl %gs:pda_irqcount
1051 cmove %rax,%rsp
Jan Beulichbd9cb642006-01-11 22:43:21 +01001052 pushq %rdx
1053 /*todo CFI_DEF_CFA_EXPRESSION ...*/
Andi Kleened6b6762005-07-28 21:15:49 -07001054 call __do_softirq
Jan Beulichbd9cb642006-01-11 22:43:21 +01001055 popq %rsp
Jan Beulich7effaa82005-09-12 18:49:24 +02001056 CFI_DEF_CFA_REGISTER rsp
Andi Kleened6b6762005-07-28 21:15:49 -07001057 decl %gs:pda_irqcount
Andi Kleened6b6762005-07-28 21:15:49 -07001058 ret
Jan Beulich7effaa82005-09-12 18:49:24 +02001059 CFI_ENDPROC