blob: 08f7e8039099dc6b8787bbad4797553d7da6ac3f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86_64/kernel/head.S -- start in 32bit and switch to 64bit
3 *
4 * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
6 * Copyright (C) 2000 Karsten Keil <kkeil@suse.de>
7 * Copyright (C) 2001,2002 Andi Kleen <ak@suse.de>
Vivek Goyal1ab60e02007-05-02 19:27:07 +02008 * Copyright (C) 2005 Eric Biederman <ebiederm@xmission.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 */
10
11
12#include <linux/linkage.h>
13#include <linux/threads.h>
Siddha, Suresh Bf6c2e332005-11-05 17:25:53 +010014#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <asm/segment.h>
Vivek Goyal67dcbb62007-05-02 19:27:06 +020016#include <asm/pgtable.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <asm/page.h>
18#include <asm/msr.h>
19#include <asm/cache.h>
Cyrill Gorcunov369101d2008-05-12 15:43:38 +020020#include <asm/processor-flags.h>
Tejun Heob12d8db2009-01-13 20:41:35 +090021#include <asm/percpu.h>
H. Peter Anvin9900aa22012-04-18 17:16:49 -070022#include <asm/nops.h>
Vivek Goyal1ab60e02007-05-02 19:27:07 +020023
Glauber de Oliveira Costa49a69782008-01-30 13:31:10 +010024#ifdef CONFIG_PARAVIRT
25#include <asm/asm-offsets.h>
26#include <asm/paravirt.h>
H. Peter Anvinffc4bc92012-04-18 17:16:48 -070027#define GET_CR2_INTO(reg) GET_CR2_INTO_RAX ; movq %rax, reg
Glauber de Oliveira Costa49a69782008-01-30 13:31:10 +010028#else
H. Peter Anvinffc4bc92012-04-18 17:16:48 -070029#define GET_CR2_INTO(reg) movq %cr2, reg
H. Peter Anvin9900aa22012-04-18 17:16:49 -070030#define INTERRUPT_RETURN iretq
Glauber de Oliveira Costa49a69782008-01-30 13:31:10 +010031#endif
32
Daniel Mack3ad2f3f2010-02-03 08:01:28 +080033/* we are not able to switch in one step to the final KERNEL ADDRESS SPACE
Vivek Goyal1ab60e02007-05-02 19:27:07 +020034 * because we need identity-mapped pages.
35 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 */
37
Eduardo Habkosta6523742008-06-25 00:19:16 -040038#define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
39
40L4_PAGE_OFFSET = pgd_index(__PAGE_OFFSET)
41L3_PAGE_OFFSET = pud_index(__PAGE_OFFSET)
42L4_START_KERNEL = pgd_index(__START_KERNEL_map)
43L3_START_KERNEL = pud_index(__START_KERNEL_map)
44
Linus Torvalds1da177e2005-04-16 15:20:36 -070045 .text
Tim Abbott4ae59b92009-09-16 16:44:28 -040046 __HEAD
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 .code64
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 .globl startup_64
49startup_64:
Vivek Goyal1ab60e02007-05-02 19:27:07 +020050 /*
Konrad Rzeszutek Wilk12562762013-02-25 15:54:10 -050051 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
Vivek Goyal1ab60e02007-05-02 19:27:07 +020052 * and someone has loaded an identity mapped page table
53 * for us. These identity mapped page tables map all of the
54 * kernel pages and possibly all of memory.
55 *
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080056 * %rsi holds a physical pointer to real_mode_data.
Vivek Goyal1ab60e02007-05-02 19:27:07 +020057 *
58 * We come here either directly from a 64bit bootloader, or from
59 * arch/x86_64/boot/compressed/head.S.
60 *
61 * We only come here initially at boot nothing else comes here.
62 *
63 * Since we may be loaded at an address different from what we were
64 * compiled to run at we first fixup the physical addresses in our page
65 * tables and then reload them.
66 */
67
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080068 /*
69 * Compute the delta between the address I am compiled to run at and the
Vivek Goyal1ab60e02007-05-02 19:27:07 +020070 * address I am actually running at.
71 */
72 leaq _text(%rip), %rbp
73 subq $_text - __START_KERNEL_map, %rbp
74
75 /* Is the address not 2M aligned? */
76 movq %rbp, %rax
Andi Kleen31422c52008-02-04 16:48:08 +010077 andl $~PMD_PAGE_MASK, %eax
Vivek Goyal1ab60e02007-05-02 19:27:07 +020078 testl %eax, %eax
79 jnz bad_address
80
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080081 /*
82 * Is the address too large?
Vivek Goyal1ab60e02007-05-02 19:27:07 +020083 */
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080084 leaq _text(%rip), %rax
85 shrq $MAX_PHYSMEM_BITS, %rax
86 jnz bad_address
Vivek Goyal1ab60e02007-05-02 19:27:07 +020087
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080088 /*
89 * Fixup the physical addresses in the page table
90 */
91 addq %rbp, early_level4_pgt + (L4_START_KERNEL*8)(%rip)
Eric W. Bidermanb1c931e2007-07-15 23:37:28 -070092
Vivek Goyal1ab60e02007-05-02 19:27:07 +020093 addq %rbp, level3_kernel_pgt + (510*8)(%rip)
Eric W. Bidermanb1c931e2007-07-15 23:37:28 -070094 addq %rbp, level3_kernel_pgt + (511*8)(%rip)
95
96 addq %rbp, level2_fixmap_pgt + (506*8)(%rip)
Vivek Goyal1ab60e02007-05-02 19:27:07 +020097
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080098 /*
99 * Set up the identity mapping for the switchover. These
100 * entries should *NOT* have the global bit set! This also
101 * creates a bunch of nonsense entries but that is fine --
102 * it avoids problems around wraparound.
103 */
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200104 leaq _text(%rip), %rdi
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800105 leaq early_level4_pgt(%rip), %rbx
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200106
107 movq %rdi, %rax
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800108 shrq $PGDIR_SHIFT, %rax
109
110 leaq (4096 + _KERNPG_TABLE)(%rbx), %rdx
111 movq %rdx, 0(%rbx,%rax,8)
112 movq %rdx, 8(%rbx,%rax,8)
113
114 addq $4096, %rdx
115 movq %rdi, %rax
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200116 shrq $PUD_SHIFT, %rax
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800117 andl $(PTRS_PER_PUD-1), %eax
118 movq %rdx, (4096+0)(%rbx,%rax,8)
119 movq %rdx, (4096+8)(%rbx,%rax,8)
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200120
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800121 addq $8192, %rbx
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200122 movq %rdi, %rax
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800123 shrq $PMD_SHIFT, %rdi
124 addq $(__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL), %rax
125 leaq (_end - 1)(%rip), %rcx
126 shrq $PMD_SHIFT, %rcx
127 subq %rdi, %rcx
128 incl %ecx
129
1301:
131 andq $(PTRS_PER_PMD - 1), %rdi
132 movq %rax, (%rbx,%rdi,8)
133 incq %rdi
134 addq $PMD_SIZE, %rax
135 decl %ecx
136 jnz 1b
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200137
Thomas Gleixner31eedd82008-02-15 17:29:12 +0100138 /*
139 * Fixup the kernel text+data virtual addresses. Note that
140 * we might write invalid pmds, when the kernel is relocated
141 * cleanup_highmap() fixes this up along with the mappings
142 * beyond _end.
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200143 */
144 leaq level2_kernel_pgt(%rip), %rdi
145 leaq 4096(%rdi), %r8
146 /* See if it is a valid page table entry */
1471: testq $1, 0(%rdi)
148 jz 2f
149 addq %rbp, 0(%rdi)
150 /* Go to the next page */
1512: addq $8, %rdi
152 cmp %r8, %rdi
153 jne 1b
154
155 /* Fixup phys_base */
156 addq %rbp, phys_base(%rip)
157
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800158 movq $(early_level4_pgt - __START_KERNEL_map), %rax
159 jmp 1f
Vivek Goyal90b1c202007-05-02 19:27:07 +0200160ENTRY(secondary_startup_64)
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200161 /*
Konrad Rzeszutek Wilk12562762013-02-25 15:54:10 -0500162 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200163 * and someone has loaded a mapped page table.
164 *
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800165 * %rsi holds a physical pointer to real_mode_data.
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200166 *
167 * We come here either from startup_64 (using physical addresses)
168 * or from trampoline.S (using virtual addresses).
169 *
170 * Using virtual addresses from trampoline.S removes the need
171 * to have any identity mapped pages in the kernel page table
172 * after the boot processor executes this code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 */
174
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800175 movq $(init_level4_pgt - __START_KERNEL_map), %rax
1761:
177
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 /* Enable PAE mode and PGE */
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800179 movl $(X86_CR4_PAE | X86_CR4_PGE), %ecx
180 movq %rcx, %cr4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
182 /* Setup early boot stage 4 level pagetables. */
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200183 addq phys_base(%rip), %rax
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 movq %rax, %cr3
185
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200186 /* Ensure I am executing from virtual addresses */
187 movq $1f, %rax
188 jmp *%rax
1891:
190
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191 /* Check if nx is implemented */
192 movl $0x80000001, %eax
193 cpuid
194 movl %edx,%edi
195
196 /* Setup EFER (Extended Feature Enable Register) */
197 movl $MSR_EFER, %ecx
198 rdmsr
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200199 btsl $_EFER_SCE, %eax /* Enable System Call */
200 btl $20,%edi /* No Execute supported? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 jnc 1f
202 btsl $_EFER_NX, %eax
H. Peter Anvin78d77df2013-05-02 10:33:46 -0700203 btsq $_PAGE_BIT_NX,early_pmd_flags(%rip)
Vivek Goyal1ab60e02007-05-02 19:27:07 +02002041: wrmsr /* Make changes effective */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205
206 /* Setup cr0 */
Cyrill Gorcunov369101d2008-05-12 15:43:38 +0200207#define CR0_STATE (X86_CR0_PE | X86_CR0_MP | X86_CR0_ET | \
208 X86_CR0_NE | X86_CR0_WP | X86_CR0_AM | \
209 X86_CR0_PG)
210 movl $CR0_STATE, %eax
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 /* Make changes effective */
212 movq %rax, %cr0
213
214 /* Setup a boot time stack */
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800215 movq stack_start(%rip), %rsp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216
217 /* zero EFLAGS after setting rsp */
218 pushq $0
219 popfq
220
221 /*
222 * We must switch to a new descriptor in kernel space for the GDT
223 * because soon the kernel won't have access anymore to the userspace
224 * addresses where we're currently running on. We have to do that here
225 * because in 32bit we couldn't load a 64bit linear address.
226 */
Glauber Costaa9390982008-05-28 16:19:53 -0700227 lgdt early_gdt_descr(%rip)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228
Brian Gerst8ec69932009-11-25 11:17:36 -0500229 /* set up data segments */
230 xorl %eax,%eax
Zachary Amsdenffb60172007-02-13 13:26:24 +0100231 movl %eax,%ds
232 movl %eax,%ss
233 movl %eax,%es
234
235 /*
236 * We don't really need to load %fs or %gs, but load them anyway
237 * to kill any stale realmode selectors. This allows execution
238 * under VT hardware.
239 */
240 movl %eax,%fs
241 movl %eax,%gs
242
Tejun Heof32ff532009-01-13 20:41:35 +0900243 /* Set up %gs.
244 *
Brian Gerst947e76c2009-01-19 12:21:28 +0900245 * The base of %gs always points to the bottom of the irqstack
246 * union. If the stack protector canary is enabled, it is
247 * located at %gs:40. Note that, on SMP, the boot cpu uses
248 * init data section till per cpu areas are set up.
Tejun Heof32ff532009-01-13 20:41:35 +0900249 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 movl $MSR_GS_BASE,%ecx
Brian Gerst650fb432010-07-17 09:03:28 -0400251 movl initial_gs(%rip),%eax
252 movl initial_gs+4(%rip),%edx
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253 wrmsr
254
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800255 /* rsi is pointer to real mode structure with interesting info.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 pass it to C */
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800257 movq %rsi, %rdi
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258
259 /* Finally jump to run C code and to be on real kernel address
260 * Since we are running on identity-mapped space we have to jump
Eric W. Biederman26374c72006-09-26 10:52:38 +0200261 * to the full 64bit address, this is only possible as indirect
262 * jump. In addition we need to ensure %cs is set so we make this
263 * a far return.
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800264 *
265 * Note: do not change to far jump indirect with 64bit offset.
266 *
267 * AMD does not support far jump indirect with 64bit offset.
268 * AMD64 Architecture Programmer's Manual, Volume 3: states only
269 * JMP FAR mem16:16 FF /5 Far jump indirect,
270 * with the target specified by a far pointer in memory.
271 * JMP FAR mem16:32 FF /5 Far jump indirect,
272 * with the target specified by a far pointer in memory.
273 *
274 * Intel64 does support 64bit offset.
275 * Software Developer Manual Vol 2: states:
276 * FF /5 JMP m16:16 Jump far, absolute indirect,
277 * address given in m16:16
278 * FF /5 JMP m16:32 Jump far, absolute indirect,
279 * address given in m16:32.
280 * REX.W + FF /5 JMP m16:64 Jump far, absolute indirect,
281 * address given in m16:64.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 */
283 movq initial_code(%rip),%rax
Eric W. Biederman26374c72006-09-26 10:52:38 +0200284 pushq $0 # fake return address to stop unwinder
285 pushq $__KERNEL_CS # set correct cs
286 pushq %rax # target address in negative space
287 lretq
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288
Fenghua Yu42e78e92012-11-13 11:32:44 -0800289#ifdef CONFIG_HOTPLUG_CPU
290/*
291 * Boot CPU0 entry point. It's called from play_dead(). Everything has been set
292 * up already except stack. We just set up stack here. Then call
293 * start_secondary().
294 */
295ENTRY(start_cpu0)
296 movq stack_start(%rip),%rsp
297 movq initial_code(%rip),%rax
298 pushq $0 # fake return address to stop unwinder
299 pushq $__KERNEL_CS # set correct cs
300 pushq %rax # target address in negative space
301 lretq
302ENDPROC(start_cpu0)
303#endif
304
Jan Beuliche57113b2006-03-25 16:30:01 +0100305 /* SMP bootup changes these two */
Sam Ravnborgda5968a2008-02-17 13:22:59 +0100306 __REFDATA
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800307 .balign 8
308 GLOBAL(initial_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 .quad x86_64_start_kernel
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800310 GLOBAL(initial_gs)
Brian Gerst2add8e22009-02-08 09:58:39 -0500311 .quad INIT_PER_CPU_VAR(irq_stack_union)
Sam Ravnborgf1fbabb2008-02-06 22:39:45 +0100312
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800313 GLOBAL(stack_start)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 .quad init_thread_union+THREAD_SIZE-8
Glauber Costa9cf4f292008-05-27 18:22:54 -0700315 .word 0
Suresh Siddhab9af7c02009-10-14 14:46:55 -0700316 __FINITDATA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200318bad_address:
319 jmp bad_address
320
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800321 __INIT
Roland McGrath8866cd92008-01-30 13:33:06 +0100322 .globl early_idt_handlers
323early_idt_handlers:
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700324 # 104(%rsp) %rflags
325 # 96(%rsp) %cs
326 # 88(%rsp) %rip
327 # 80(%rsp) error code
Andi Kleen749c9702008-03-11 02:23:22 +0100328 i = 0
329 .rept NUM_EXCEPTION_VECTORS
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700330 .if (EXCEPTION_ERRCODE_MASK >> i) & 1
331 ASM_NOP2
332 .else
333 pushq $0 # Dummy error code, to make stack frame uniform
334 .endif
335 pushq $i # 72(%rsp) Vector number
Andi Kleen749c9702008-03-11 02:23:22 +0100336 jmp early_idt_handler
337 i = i + 1
338 .endr
Roland McGrath8866cd92008-01-30 13:33:06 +0100339
Linus Torvaldsac630dd2013-02-22 13:09:51 -0800340/* This is global to keep gas from relaxing the jumps */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341ENTRY(early_idt_handler)
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700342 cld
343
Andi Kleenb9575912005-04-16 15:25:00 -0700344 cmpl $2,early_recursion_flag(%rip)
345 jz 1f
346 incl early_recursion_flag(%rip)
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700347
348 pushq %rax # 64(%rsp)
349 pushq %rcx # 56(%rsp)
350 pushq %rdx # 48(%rsp)
351 pushq %rsi # 40(%rsp)
352 pushq %rdi # 32(%rsp)
353 pushq %r8 # 24(%rsp)
354 pushq %r9 # 16(%rsp)
355 pushq %r10 # 8(%rsp)
356 pushq %r11 # 0(%rsp)
357
358 cmpl $__KERNEL_CS,96(%rsp)
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800359 jne 11f
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700360
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800361 cmpl $14,72(%rsp) # Page fault?
362 jnz 10f
363 GET_CR2_INTO(%rdi) # can clobber any volatile register if pv
364 call early_make_pgtable
365 andl %eax,%eax
366 jz 20f # All good
367
36810:
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700369 leaq 88(%rsp),%rdi # Pointer to %rip
370 call early_fixup_exception
371 andl %eax,%eax
372 jnz 20f # Found an exception entry
373
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080037411:
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700375#ifdef CONFIG_EARLY_PRINTK
376 GET_CR2_INTO(%r9) # can clobber any volatile register if pv
377 movl 80(%rsp),%r8d # error code
378 movl 72(%rsp),%esi # vector number
379 movl 96(%rsp),%edx # %cs
380 movq 88(%rsp),%rcx # %rip
Roland McGrath8866cd92008-01-30 13:33:06 +0100381 xorl %eax,%eax
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 leaq early_idt_msg(%rip),%rdi
383 call early_printk
Andi Kleenb9575912005-04-16 15:25:00 -0700384 cmpl $2,early_recursion_flag(%rip)
385 jz 1f
386 call dump_stack
Andi Kleen6574ffd2006-02-16 23:42:10 +0100387#ifdef CONFIG_KALLSYMS
388 leaq early_idt_ripmsg(%rip),%rdi
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700389 movq 40(%rsp),%rsi # %rip again
Andi Kleen6574ffd2006-02-16 23:42:10 +0100390 call __print_symbol
391#endif
Ingo Molnar076f9772008-01-30 13:33:06 +0100392#endif /* EARLY_PRINTK */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003931: hlt
394 jmp 1b
Ingo Molnar076f9772008-01-30 13:33:06 +0100395
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080039620: # Exception table entry found or page table generated
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700397 popq %r11
398 popq %r10
399 popq %r9
400 popq %r8
401 popq %rdi
402 popq %rsi
403 popq %rdx
404 popq %rcx
405 popq %rax
406 addq $16,%rsp # drop vector number and error code
407 decl early_recursion_flag(%rip)
408 INTERRUPT_RETURN
Linus Torvaldsac630dd2013-02-22 13:09:51 -0800409ENDPROC(early_idt_handler)
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700410
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800411 __INITDATA
412
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700413 .balign 4
Andi Kleenb9575912005-04-16 15:25:00 -0700414early_recursion_flag:
415 .long 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416
H. Peter Anvin9900aa22012-04-18 17:16:49 -0700417#ifdef CONFIG_EARLY_PRINTK
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418early_idt_msg:
Roland McGrath8866cd92008-01-30 13:33:06 +0100419 .asciz "PANIC: early exception %02lx rip %lx:%lx error %lx cr2 %lx\n"
Andi Kleen6574ffd2006-02-16 23:42:10 +0100420early_idt_ripmsg:
421 .asciz "RIP %s\n"
Ingo Molnar076f9772008-01-30 13:33:06 +0100422#endif /* CONFIG_EARLY_PRINTK */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423
Jan Beulichf0cf5d12006-01-17 07:03:32 +0100424#define NEXT_PAGE(name) \
Vivek Goyal67dcbb62007-05-02 19:27:06 +0200425 .balign PAGE_SIZE; \
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800426GLOBAL(name)
Jan Beulichf0cf5d12006-01-17 07:03:32 +0100427
Vivek Goyal67dcbb62007-05-02 19:27:06 +0200428/* Automate the creation of 1 to 1 mapping pmd entries */
Cyrill Gorcunov0e192b92008-05-13 20:55:40 +0400429#define PMDS(START, PERM, COUNT) \
430 i = 0 ; \
431 .rept (COUNT) ; \
432 .quad (START) + (i << PMD_SHIFT) + (PERM) ; \
433 i = i + 1 ; \
Vivek Goyal67dcbb62007-05-02 19:27:06 +0200434 .endr
435
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800436 __INITDATA
437NEXT_PAGE(early_level4_pgt)
438 .fill 511,8,0
Vivek Goyalcfd243d2007-05-02 19:27:07 +0200439 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800441NEXT_PAGE(early_dynamic_pgts)
442 .fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0
443
444 .data
445
446#ifndef CONFIG_XEN
447NEXT_PAGE(init_level4_pgt)
448 .fill 512,8,0
449#else
450NEXT_PAGE(init_level4_pgt)
451 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
452 .org init_level4_pgt + L4_PAGE_OFFSET*8, 0
453 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
454 .org init_level4_pgt + L4_START_KERNEL*8, 0
455 /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
456 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
457
Jan Beulichf0cf5d12006-01-17 07:03:32 +0100458NEXT_PAGE(level3_ident_pgt)
Vivek Goyal67dcbb62007-05-02 19:27:06 +0200459 .quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800460 .fill 511, 8, 0
461NEXT_PAGE(level2_ident_pgt)
462 /* Since I easily can, map the first 1G.
463 * Don't set NX because code runs from these pages.
464 */
465 PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
466#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467
Jan Beulichf0cf5d12006-01-17 07:03:32 +0100468NEXT_PAGE(level3_kernel_pgt)
Eduardo Habkosta6523742008-06-25 00:19:16 -0400469 .fill L3_START_KERNEL,8,0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */
Vivek Goyal67dcbb62007-05-02 19:27:06 +0200471 .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE
Eric W. Bidermanb1c931e2007-07-15 23:37:28 -0700472 .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE
473
Jan Beulichf0cf5d12006-01-17 07:03:32 +0100474NEXT_PAGE(level2_kernel_pgt)
Ingo Molnar88f3aec2008-02-21 11:04:11 +0100475 /*
Ingo Molnar85eb69a2008-02-21 12:50:51 +0100476 * 512 MB kernel mapping. We spend a full page on this pagetable
Ingo Molnar88f3aec2008-02-21 11:04:11 +0100477 * anyway.
478 *
479 * The kernel code+data+bss must not be bigger than that.
480 *
Ingo Molnar85eb69a2008-02-21 12:50:51 +0100481 * (NOTE: at +512MB starts the module area, see MODULES_VADDR.
Ingo Molnar88f3aec2008-02-21 11:04:11 +0100482 * If you want to increase this then increase MODULES_VADDR
483 * too.)
484 */
Jeremy Fitzhardinge84906382008-07-01 16:46:35 -0700485 PMDS(0, __PAGE_KERNEL_LARGE_EXEC,
Ingo Molnard4afe412008-02-21 13:39:30 +0100486 KERNEL_IMAGE_SIZE/PMD_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800488NEXT_PAGE(level2_fixmap_pgt)
489 .fill 506,8,0
490 .quad level1_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE
491 /* 8MB reserved for vsyscalls + a 2MB hole = 4 + 1 entries */
492 .fill 5,8,0
493
494NEXT_PAGE(level1_fixmap_pgt)
495 .fill 512,8,0
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200496
Vivek Goyal67dcbb62007-05-02 19:27:06 +0200497#undef PMDS
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498
Jan Beulichf0cf5d12006-01-17 07:03:32 +0100499 .data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 .align 16
Glauber Costaa9390982008-05-28 16:19:53 -0700501 .globl early_gdt_descr
502early_gdt_descr:
503 .word GDT_ENTRIES*8-1
Tejun Heo3e5d8f92009-01-13 20:41:35 +0900504early_gdt_descr_base:
Brian Gerst2add8e22009-02-08 09:58:39 -0500505 .quad INIT_PER_CPU_VAR(gdt_page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506
Vivek Goyal1ab60e02007-05-02 19:27:07 +0200507ENTRY(phys_base)
508 /* This must match the first entry in level2_kernel_pgt */
509 .quad 0x0000000000000000
510
Jeremy Fitzhardinge8c5e5ac2008-07-08 15:06:44 -0700511#include "../../x86/xen/xen-head.S"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512
Jan Beuliche57113b2006-03-25 16:30:01 +0100513 .section .bss, "aw", @nobits
514 .align L1_CACHE_BYTES
515ENTRY(idt_table)
Cyrill Gorcunov5e112ae2009-02-23 22:56:58 +0300516 .skip IDT_ENTRIES * 16
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517
Steven Rostedt228bdaa2011-12-09 03:02:19 -0500518 .align L1_CACHE_BYTES
519ENTRY(nmi_idt_table)
520 .skip IDT_ENTRIES * 16
521
Tim Abbott02b7da32009-09-20 18:14:14 -0400522 __PAGE_ALIGNED_BSS
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800523NEXT_PAGE(empty_zero_page)
Jan Beuliche57113b2006-03-25 16:30:01 +0100524 .skip PAGE_SIZE