Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/arch/x86_64/kernel/head.S -- start in 32bit and switch to 64bit |
| 3 | * |
| 4 | * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE |
| 5 | * Copyright (C) 2000 Pavel Machek <pavel@suse.cz> |
| 6 | * Copyright (C) 2000 Karsten Keil <kkeil@suse.de> |
| 7 | * Copyright (C) 2001,2002 Andi Kleen <ak@suse.de> |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 8 | * Copyright (C) 2005 Eric Biederman <ebiederm@xmission.com> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9 | */ |
| 10 | |
| 11 | |
| 12 | #include <linux/linkage.h> |
| 13 | #include <linux/threads.h> |
Siddha, Suresh B | f6c2e33 | 2005-11-05 17:25:53 +0100 | [diff] [blame] | 14 | #include <linux/init.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | #include <asm/segment.h> |
Vivek Goyal | 67dcbb6 | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 16 | #include <asm/pgtable.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #include <asm/page.h> |
| 18 | #include <asm/msr.h> |
| 19 | #include <asm/cache.h> |
Cyrill Gorcunov | 369101d | 2008-05-12 15:43:38 +0200 | [diff] [blame] | 20 | #include <asm/processor-flags.h> |
Tejun Heo | b12d8db | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 21 | #include <asm/percpu.h> |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 22 | #include <asm/nops.h> |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 23 | |
Glauber de Oliveira Costa | 49a6978 | 2008-01-30 13:31:10 +0100 | [diff] [blame] | 24 | #ifdef CONFIG_PARAVIRT |
| 25 | #include <asm/asm-offsets.h> |
| 26 | #include <asm/paravirt.h> |
H. Peter Anvin | ffc4bc9 | 2012-04-18 17:16:48 -0700 | [diff] [blame] | 27 | #define GET_CR2_INTO(reg) GET_CR2_INTO_RAX ; movq %rax, reg |
Glauber de Oliveira Costa | 49a6978 | 2008-01-30 13:31:10 +0100 | [diff] [blame] | 28 | #else |
H. Peter Anvin | ffc4bc9 | 2012-04-18 17:16:48 -0700 | [diff] [blame] | 29 | #define GET_CR2_INTO(reg) movq %cr2, reg |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 30 | #define INTERRUPT_RETURN iretq |
Glauber de Oliveira Costa | 49a6978 | 2008-01-30 13:31:10 +0100 | [diff] [blame] | 31 | #endif |
| 32 | |
Daniel Mack | 3ad2f3f | 2010-02-03 08:01:28 +0800 | [diff] [blame] | 33 | /* we are not able to switch in one step to the final KERNEL ADDRESS SPACE |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 34 | * because we need identity-mapped pages. |
| 35 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 36 | */ |
| 37 | |
Eduardo Habkost | a652374 | 2008-06-25 00:19:16 -0400 | [diff] [blame] | 38 | #define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) |
| 39 | |
| 40 | L4_PAGE_OFFSET = pgd_index(__PAGE_OFFSET) |
| 41 | L3_PAGE_OFFSET = pud_index(__PAGE_OFFSET) |
| 42 | L4_START_KERNEL = pgd_index(__START_KERNEL_map) |
| 43 | L3_START_KERNEL = pud_index(__START_KERNEL_map) |
| 44 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | .text |
Tim Abbott | 4ae59b9 | 2009-09-16 16:44:28 -0400 | [diff] [blame] | 46 | __HEAD |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | .code64 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 48 | .globl startup_64 |
| 49 | startup_64: |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 50 | /* |
Konrad Rzeszutek Wilk | 1256276 | 2013-02-25 15:54:10 -0500 | [diff] [blame] | 51 | * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 52 | * and someone has loaded an identity mapped page table |
| 53 | * for us. These identity mapped page tables map all of the |
| 54 | * kernel pages and possibly all of memory. |
| 55 | * |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 56 | * %rsi holds a physical pointer to real_mode_data. |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 57 | * |
| 58 | * We come here either directly from a 64bit bootloader, or from |
| 59 | * arch/x86_64/boot/compressed/head.S. |
| 60 | * |
| 61 | * We only come here initially at boot nothing else comes here. |
| 62 | * |
| 63 | * Since we may be loaded at an address different from what we were |
| 64 | * compiled to run at we first fixup the physical addresses in our page |
| 65 | * tables and then reload them. |
| 66 | */ |
| 67 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 68 | /* |
| 69 | * Compute the delta between the address I am compiled to run at and the |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 70 | * address I am actually running at. |
| 71 | */ |
| 72 | leaq _text(%rip), %rbp |
| 73 | subq $_text - __START_KERNEL_map, %rbp |
| 74 | |
| 75 | /* Is the address not 2M aligned? */ |
| 76 | movq %rbp, %rax |
Andi Kleen | 31422c5 | 2008-02-04 16:48:08 +0100 | [diff] [blame] | 77 | andl $~PMD_PAGE_MASK, %eax |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 78 | testl %eax, %eax |
| 79 | jnz bad_address |
| 80 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 81 | /* |
| 82 | * Is the address too large? |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 83 | */ |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 84 | leaq _text(%rip), %rax |
| 85 | shrq $MAX_PHYSMEM_BITS, %rax |
| 86 | jnz bad_address |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 87 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 88 | /* |
| 89 | * Fixup the physical addresses in the page table |
| 90 | */ |
| 91 | addq %rbp, early_level4_pgt + (L4_START_KERNEL*8)(%rip) |
Eric W. Biderman | b1c931e | 2007-07-15 23:37:28 -0700 | [diff] [blame] | 92 | |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 93 | addq %rbp, level3_kernel_pgt + (510*8)(%rip) |
Eric W. Biderman | b1c931e | 2007-07-15 23:37:28 -0700 | [diff] [blame] | 94 | addq %rbp, level3_kernel_pgt + (511*8)(%rip) |
| 95 | |
| 96 | addq %rbp, level2_fixmap_pgt + (506*8)(%rip) |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 97 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 98 | /* |
| 99 | * Set up the identity mapping for the switchover. These |
| 100 | * entries should *NOT* have the global bit set! This also |
| 101 | * creates a bunch of nonsense entries but that is fine -- |
| 102 | * it avoids problems around wraparound. |
| 103 | */ |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 104 | leaq _text(%rip), %rdi |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 105 | leaq early_level4_pgt(%rip), %rbx |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 106 | |
| 107 | movq %rdi, %rax |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 108 | shrq $PGDIR_SHIFT, %rax |
| 109 | |
| 110 | leaq (4096 + _KERNPG_TABLE)(%rbx), %rdx |
| 111 | movq %rdx, 0(%rbx,%rax,8) |
| 112 | movq %rdx, 8(%rbx,%rax,8) |
| 113 | |
| 114 | addq $4096, %rdx |
| 115 | movq %rdi, %rax |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 116 | shrq $PUD_SHIFT, %rax |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 117 | andl $(PTRS_PER_PUD-1), %eax |
| 118 | movq %rdx, (4096+0)(%rbx,%rax,8) |
| 119 | movq %rdx, (4096+8)(%rbx,%rax,8) |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 120 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 121 | addq $8192, %rbx |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 122 | movq %rdi, %rax |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 123 | shrq $PMD_SHIFT, %rdi |
| 124 | addq $(__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL), %rax |
| 125 | leaq (_end - 1)(%rip), %rcx |
| 126 | shrq $PMD_SHIFT, %rcx |
| 127 | subq %rdi, %rcx |
| 128 | incl %ecx |
| 129 | |
| 130 | 1: |
| 131 | andq $(PTRS_PER_PMD - 1), %rdi |
| 132 | movq %rax, (%rbx,%rdi,8) |
| 133 | incq %rdi |
| 134 | addq $PMD_SIZE, %rax |
| 135 | decl %ecx |
| 136 | jnz 1b |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 137 | |
Thomas Gleixner | 31eedd8 | 2008-02-15 17:29:12 +0100 | [diff] [blame] | 138 | /* |
| 139 | * Fixup the kernel text+data virtual addresses. Note that |
| 140 | * we might write invalid pmds, when the kernel is relocated |
| 141 | * cleanup_highmap() fixes this up along with the mappings |
| 142 | * beyond _end. |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 143 | */ |
| 144 | leaq level2_kernel_pgt(%rip), %rdi |
| 145 | leaq 4096(%rdi), %r8 |
| 146 | /* See if it is a valid page table entry */ |
| 147 | 1: testq $1, 0(%rdi) |
| 148 | jz 2f |
| 149 | addq %rbp, 0(%rdi) |
| 150 | /* Go to the next page */ |
| 151 | 2: addq $8, %rdi |
| 152 | cmp %r8, %rdi |
| 153 | jne 1b |
| 154 | |
| 155 | /* Fixup phys_base */ |
| 156 | addq %rbp, phys_base(%rip) |
| 157 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 158 | movq $(early_level4_pgt - __START_KERNEL_map), %rax |
| 159 | jmp 1f |
Vivek Goyal | 90b1c20 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 160 | ENTRY(secondary_startup_64) |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 161 | /* |
Konrad Rzeszutek Wilk | 1256276 | 2013-02-25 15:54:10 -0500 | [diff] [blame] | 162 | * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 163 | * and someone has loaded a mapped page table. |
| 164 | * |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 165 | * %rsi holds a physical pointer to real_mode_data. |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 166 | * |
| 167 | * We come here either from startup_64 (using physical addresses) |
| 168 | * or from trampoline.S (using virtual addresses). |
| 169 | * |
| 170 | * Using virtual addresses from trampoline.S removes the need |
| 171 | * to have any identity mapped pages in the kernel page table |
| 172 | * after the boot processor executes this code. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 173 | */ |
| 174 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 175 | movq $(init_level4_pgt - __START_KERNEL_map), %rax |
| 176 | 1: |
| 177 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 178 | /* Enable PAE mode and PGE */ |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 179 | movl $(X86_CR4_PAE | X86_CR4_PGE), %ecx |
| 180 | movq %rcx, %cr4 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 181 | |
| 182 | /* Setup early boot stage 4 level pagetables. */ |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 183 | addq phys_base(%rip), %rax |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 184 | movq %rax, %cr3 |
| 185 | |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 186 | /* Ensure I am executing from virtual addresses */ |
| 187 | movq $1f, %rax |
| 188 | jmp *%rax |
| 189 | 1: |
| 190 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 191 | /* Check if nx is implemented */ |
| 192 | movl $0x80000001, %eax |
| 193 | cpuid |
| 194 | movl %edx,%edi |
| 195 | |
| 196 | /* Setup EFER (Extended Feature Enable Register) */ |
| 197 | movl $MSR_EFER, %ecx |
| 198 | rdmsr |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 199 | btsl $_EFER_SCE, %eax /* Enable System Call */ |
| 200 | btl $20,%edi /* No Execute supported? */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 201 | jnc 1f |
| 202 | btsl $_EFER_NX, %eax |
H. Peter Anvin | 78d77df | 2013-05-02 10:33:46 -0700 | [diff] [blame] | 203 | btsq $_PAGE_BIT_NX,early_pmd_flags(%rip) |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 204 | 1: wrmsr /* Make changes effective */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 205 | |
| 206 | /* Setup cr0 */ |
Cyrill Gorcunov | 369101d | 2008-05-12 15:43:38 +0200 | [diff] [blame] | 207 | #define CR0_STATE (X86_CR0_PE | X86_CR0_MP | X86_CR0_ET | \ |
| 208 | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM | \ |
| 209 | X86_CR0_PG) |
| 210 | movl $CR0_STATE, %eax |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 211 | /* Make changes effective */ |
| 212 | movq %rax, %cr0 |
| 213 | |
| 214 | /* Setup a boot time stack */ |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 215 | movq stack_start(%rip), %rsp |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 216 | |
| 217 | /* zero EFLAGS after setting rsp */ |
| 218 | pushq $0 |
| 219 | popfq |
| 220 | |
| 221 | /* |
| 222 | * We must switch to a new descriptor in kernel space for the GDT |
| 223 | * because soon the kernel won't have access anymore to the userspace |
| 224 | * addresses where we're currently running on. We have to do that here |
| 225 | * because in 32bit we couldn't load a 64bit linear address. |
| 226 | */ |
Glauber Costa | a939098 | 2008-05-28 16:19:53 -0700 | [diff] [blame] | 227 | lgdt early_gdt_descr(%rip) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 228 | |
Brian Gerst | 8ec6993 | 2009-11-25 11:17:36 -0500 | [diff] [blame] | 229 | /* set up data segments */ |
| 230 | xorl %eax,%eax |
Zachary Amsden | ffb6017 | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 231 | movl %eax,%ds |
| 232 | movl %eax,%ss |
| 233 | movl %eax,%es |
| 234 | |
| 235 | /* |
| 236 | * We don't really need to load %fs or %gs, but load them anyway |
| 237 | * to kill any stale realmode selectors. This allows execution |
| 238 | * under VT hardware. |
| 239 | */ |
| 240 | movl %eax,%fs |
| 241 | movl %eax,%gs |
| 242 | |
Tejun Heo | f32ff53 | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 243 | /* Set up %gs. |
| 244 | * |
Brian Gerst | 947e76c | 2009-01-19 12:21:28 +0900 | [diff] [blame] | 245 | * The base of %gs always points to the bottom of the irqstack |
| 246 | * union. If the stack protector canary is enabled, it is |
| 247 | * located at %gs:40. Note that, on SMP, the boot cpu uses |
| 248 | * init data section till per cpu areas are set up. |
Tejun Heo | f32ff53 | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 249 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 250 | movl $MSR_GS_BASE,%ecx |
Brian Gerst | 650fb43 | 2010-07-17 09:03:28 -0400 | [diff] [blame] | 251 | movl initial_gs(%rip),%eax |
| 252 | movl initial_gs+4(%rip),%edx |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 253 | wrmsr |
| 254 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 255 | /* rsi is pointer to real mode structure with interesting info. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 256 | pass it to C */ |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 257 | movq %rsi, %rdi |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 258 | |
| 259 | /* Finally jump to run C code and to be on real kernel address |
| 260 | * Since we are running on identity-mapped space we have to jump |
Eric W. Biederman | 26374c7 | 2006-09-26 10:52:38 +0200 | [diff] [blame] | 261 | * to the full 64bit address, this is only possible as indirect |
| 262 | * jump. In addition we need to ensure %cs is set so we make this |
| 263 | * a far return. |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 264 | * |
| 265 | * Note: do not change to far jump indirect with 64bit offset. |
| 266 | * |
| 267 | * AMD does not support far jump indirect with 64bit offset. |
| 268 | * AMD64 Architecture Programmer's Manual, Volume 3: states only |
| 269 | * JMP FAR mem16:16 FF /5 Far jump indirect, |
| 270 | * with the target specified by a far pointer in memory. |
| 271 | * JMP FAR mem16:32 FF /5 Far jump indirect, |
| 272 | * with the target specified by a far pointer in memory. |
| 273 | * |
| 274 | * Intel64 does support 64bit offset. |
| 275 | * Software Developer Manual Vol 2: states: |
| 276 | * FF /5 JMP m16:16 Jump far, absolute indirect, |
| 277 | * address given in m16:16 |
| 278 | * FF /5 JMP m16:32 Jump far, absolute indirect, |
| 279 | * address given in m16:32. |
| 280 | * REX.W + FF /5 JMP m16:64 Jump far, absolute indirect, |
| 281 | * address given in m16:64. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 282 | */ |
| 283 | movq initial_code(%rip),%rax |
Eric W. Biederman | 26374c7 | 2006-09-26 10:52:38 +0200 | [diff] [blame] | 284 | pushq $0 # fake return address to stop unwinder |
| 285 | pushq $__KERNEL_CS # set correct cs |
| 286 | pushq %rax # target address in negative space |
| 287 | lretq |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 288 | |
Fenghua Yu | 42e78e9 | 2012-11-13 11:32:44 -0800 | [diff] [blame] | 289 | #ifdef CONFIG_HOTPLUG_CPU |
| 290 | /* |
| 291 | * Boot CPU0 entry point. It's called from play_dead(). Everything has been set |
| 292 | * up already except stack. We just set up stack here. Then call |
| 293 | * start_secondary(). |
| 294 | */ |
| 295 | ENTRY(start_cpu0) |
| 296 | movq stack_start(%rip),%rsp |
| 297 | movq initial_code(%rip),%rax |
| 298 | pushq $0 # fake return address to stop unwinder |
| 299 | pushq $__KERNEL_CS # set correct cs |
| 300 | pushq %rax # target address in negative space |
| 301 | lretq |
| 302 | ENDPROC(start_cpu0) |
| 303 | #endif |
| 304 | |
Jan Beulich | e57113b | 2006-03-25 16:30:01 +0100 | [diff] [blame] | 305 | /* SMP bootup changes these two */ |
Sam Ravnborg | da5968a | 2008-02-17 13:22:59 +0100 | [diff] [blame] | 306 | __REFDATA |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 307 | .balign 8 |
| 308 | GLOBAL(initial_code) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 309 | .quad x86_64_start_kernel |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 310 | GLOBAL(initial_gs) |
Brian Gerst | 2add8e2 | 2009-02-08 09:58:39 -0500 | [diff] [blame] | 311 | .quad INIT_PER_CPU_VAR(irq_stack_union) |
Sam Ravnborg | f1fbabb | 2008-02-06 22:39:45 +0100 | [diff] [blame] | 312 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 313 | GLOBAL(stack_start) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 314 | .quad init_thread_union+THREAD_SIZE-8 |
Glauber Costa | 9cf4f29 | 2008-05-27 18:22:54 -0700 | [diff] [blame] | 315 | .word 0 |
Suresh Siddha | b9af7c0 | 2009-10-14 14:46:55 -0700 | [diff] [blame] | 316 | __FINITDATA |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 317 | |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 318 | bad_address: |
| 319 | jmp bad_address |
| 320 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 321 | __INIT |
Roland McGrath | 8866cd9 | 2008-01-30 13:33:06 +0100 | [diff] [blame] | 322 | .globl early_idt_handlers |
| 323 | early_idt_handlers: |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 324 | # 104(%rsp) %rflags |
| 325 | # 96(%rsp) %cs |
| 326 | # 88(%rsp) %rip |
| 327 | # 80(%rsp) error code |
Andi Kleen | 749c970 | 2008-03-11 02:23:22 +0100 | [diff] [blame] | 328 | i = 0 |
| 329 | .rept NUM_EXCEPTION_VECTORS |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 330 | .if (EXCEPTION_ERRCODE_MASK >> i) & 1 |
| 331 | ASM_NOP2 |
| 332 | .else |
| 333 | pushq $0 # Dummy error code, to make stack frame uniform |
| 334 | .endif |
| 335 | pushq $i # 72(%rsp) Vector number |
Andi Kleen | 749c970 | 2008-03-11 02:23:22 +0100 | [diff] [blame] | 336 | jmp early_idt_handler |
| 337 | i = i + 1 |
| 338 | .endr |
Roland McGrath | 8866cd9 | 2008-01-30 13:33:06 +0100 | [diff] [blame] | 339 | |
Linus Torvalds | ac630dd | 2013-02-22 13:09:51 -0800 | [diff] [blame] | 340 | /* This is global to keep gas from relaxing the jumps */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 341 | ENTRY(early_idt_handler) |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 342 | cld |
| 343 | |
Andi Kleen | b957591 | 2005-04-16 15:25:00 -0700 | [diff] [blame] | 344 | cmpl $2,early_recursion_flag(%rip) |
| 345 | jz 1f |
| 346 | incl early_recursion_flag(%rip) |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 347 | |
| 348 | pushq %rax # 64(%rsp) |
| 349 | pushq %rcx # 56(%rsp) |
| 350 | pushq %rdx # 48(%rsp) |
| 351 | pushq %rsi # 40(%rsp) |
| 352 | pushq %rdi # 32(%rsp) |
| 353 | pushq %r8 # 24(%rsp) |
| 354 | pushq %r9 # 16(%rsp) |
| 355 | pushq %r10 # 8(%rsp) |
| 356 | pushq %r11 # 0(%rsp) |
| 357 | |
| 358 | cmpl $__KERNEL_CS,96(%rsp) |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 359 | jne 11f |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 360 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 361 | cmpl $14,72(%rsp) # Page fault? |
| 362 | jnz 10f |
| 363 | GET_CR2_INTO(%rdi) # can clobber any volatile register if pv |
| 364 | call early_make_pgtable |
| 365 | andl %eax,%eax |
| 366 | jz 20f # All good |
| 367 | |
| 368 | 10: |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 369 | leaq 88(%rsp),%rdi # Pointer to %rip |
| 370 | call early_fixup_exception |
| 371 | andl %eax,%eax |
| 372 | jnz 20f # Found an exception entry |
| 373 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 374 | 11: |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 375 | #ifdef CONFIG_EARLY_PRINTK |
| 376 | GET_CR2_INTO(%r9) # can clobber any volatile register if pv |
| 377 | movl 80(%rsp),%r8d # error code |
| 378 | movl 72(%rsp),%esi # vector number |
| 379 | movl 96(%rsp),%edx # %cs |
| 380 | movq 88(%rsp),%rcx # %rip |
Roland McGrath | 8866cd9 | 2008-01-30 13:33:06 +0100 | [diff] [blame] | 381 | xorl %eax,%eax |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 382 | leaq early_idt_msg(%rip),%rdi |
| 383 | call early_printk |
Andi Kleen | b957591 | 2005-04-16 15:25:00 -0700 | [diff] [blame] | 384 | cmpl $2,early_recursion_flag(%rip) |
| 385 | jz 1f |
| 386 | call dump_stack |
Andi Kleen | 6574ffd | 2006-02-16 23:42:10 +0100 | [diff] [blame] | 387 | #ifdef CONFIG_KALLSYMS |
| 388 | leaq early_idt_ripmsg(%rip),%rdi |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 389 | movq 40(%rsp),%rsi # %rip again |
Andi Kleen | 6574ffd | 2006-02-16 23:42:10 +0100 | [diff] [blame] | 390 | call __print_symbol |
| 391 | #endif |
Ingo Molnar | 076f977 | 2008-01-30 13:33:06 +0100 | [diff] [blame] | 392 | #endif /* EARLY_PRINTK */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 393 | 1: hlt |
| 394 | jmp 1b |
Ingo Molnar | 076f977 | 2008-01-30 13:33:06 +0100 | [diff] [blame] | 395 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 396 | 20: # Exception table entry found or page table generated |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 397 | popq %r11 |
| 398 | popq %r10 |
| 399 | popq %r9 |
| 400 | popq %r8 |
| 401 | popq %rdi |
| 402 | popq %rsi |
| 403 | popq %rdx |
| 404 | popq %rcx |
| 405 | popq %rax |
| 406 | addq $16,%rsp # drop vector number and error code |
| 407 | decl early_recursion_flag(%rip) |
| 408 | INTERRUPT_RETURN |
Linus Torvalds | ac630dd | 2013-02-22 13:09:51 -0800 | [diff] [blame] | 409 | ENDPROC(early_idt_handler) |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 410 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 411 | __INITDATA |
| 412 | |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 413 | .balign 4 |
Andi Kleen | b957591 | 2005-04-16 15:25:00 -0700 | [diff] [blame] | 414 | early_recursion_flag: |
| 415 | .long 0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 416 | |
H. Peter Anvin | 9900aa2 | 2012-04-18 17:16:49 -0700 | [diff] [blame] | 417 | #ifdef CONFIG_EARLY_PRINTK |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 418 | early_idt_msg: |
Roland McGrath | 8866cd9 | 2008-01-30 13:33:06 +0100 | [diff] [blame] | 419 | .asciz "PANIC: early exception %02lx rip %lx:%lx error %lx cr2 %lx\n" |
Andi Kleen | 6574ffd | 2006-02-16 23:42:10 +0100 | [diff] [blame] | 420 | early_idt_ripmsg: |
| 421 | .asciz "RIP %s\n" |
Ingo Molnar | 076f977 | 2008-01-30 13:33:06 +0100 | [diff] [blame] | 422 | #endif /* CONFIG_EARLY_PRINTK */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 423 | |
Jan Beulich | f0cf5d1 | 2006-01-17 07:03:32 +0100 | [diff] [blame] | 424 | #define NEXT_PAGE(name) \ |
Vivek Goyal | 67dcbb6 | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 425 | .balign PAGE_SIZE; \ |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 426 | GLOBAL(name) |
Jan Beulich | f0cf5d1 | 2006-01-17 07:03:32 +0100 | [diff] [blame] | 427 | |
Vivek Goyal | 67dcbb6 | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 428 | /* Automate the creation of 1 to 1 mapping pmd entries */ |
Cyrill Gorcunov | 0e192b9 | 2008-05-13 20:55:40 +0400 | [diff] [blame] | 429 | #define PMDS(START, PERM, COUNT) \ |
| 430 | i = 0 ; \ |
| 431 | .rept (COUNT) ; \ |
| 432 | .quad (START) + (i << PMD_SHIFT) + (PERM) ; \ |
| 433 | i = i + 1 ; \ |
Vivek Goyal | 67dcbb6 | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 434 | .endr |
| 435 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 436 | __INITDATA |
| 437 | NEXT_PAGE(early_level4_pgt) |
| 438 | .fill 511,8,0 |
Vivek Goyal | cfd243d | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 439 | .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 440 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 441 | NEXT_PAGE(early_dynamic_pgts) |
| 442 | .fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0 |
| 443 | |
| 444 | .data |
| 445 | |
| 446 | #ifndef CONFIG_XEN |
| 447 | NEXT_PAGE(init_level4_pgt) |
| 448 | .fill 512,8,0 |
| 449 | #else |
| 450 | NEXT_PAGE(init_level4_pgt) |
| 451 | .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE |
| 452 | .org init_level4_pgt + L4_PAGE_OFFSET*8, 0 |
| 453 | .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE |
| 454 | .org init_level4_pgt + L4_START_KERNEL*8, 0 |
| 455 | /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */ |
| 456 | .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE |
| 457 | |
Jan Beulich | f0cf5d1 | 2006-01-17 07:03:32 +0100 | [diff] [blame] | 458 | NEXT_PAGE(level3_ident_pgt) |
Vivek Goyal | 67dcbb6 | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 459 | .quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 460 | .fill 511, 8, 0 |
| 461 | NEXT_PAGE(level2_ident_pgt) |
| 462 | /* Since I easily can, map the first 1G. |
| 463 | * Don't set NX because code runs from these pages. |
| 464 | */ |
| 465 | PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD) |
| 466 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 467 | |
Jan Beulich | f0cf5d1 | 2006-01-17 07:03:32 +0100 | [diff] [blame] | 468 | NEXT_PAGE(level3_kernel_pgt) |
Eduardo Habkost | a652374 | 2008-06-25 00:19:16 -0400 | [diff] [blame] | 469 | .fill L3_START_KERNEL,8,0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 470 | /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */ |
Vivek Goyal | 67dcbb6 | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 471 | .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE |
Eric W. Biderman | b1c931e | 2007-07-15 23:37:28 -0700 | [diff] [blame] | 472 | .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE |
| 473 | |
Jan Beulich | f0cf5d1 | 2006-01-17 07:03:32 +0100 | [diff] [blame] | 474 | NEXT_PAGE(level2_kernel_pgt) |
Ingo Molnar | 88f3aec | 2008-02-21 11:04:11 +0100 | [diff] [blame] | 475 | /* |
Ingo Molnar | 85eb69a | 2008-02-21 12:50:51 +0100 | [diff] [blame] | 476 | * 512 MB kernel mapping. We spend a full page on this pagetable |
Ingo Molnar | 88f3aec | 2008-02-21 11:04:11 +0100 | [diff] [blame] | 477 | * anyway. |
| 478 | * |
| 479 | * The kernel code+data+bss must not be bigger than that. |
| 480 | * |
Ingo Molnar | 85eb69a | 2008-02-21 12:50:51 +0100 | [diff] [blame] | 481 | * (NOTE: at +512MB starts the module area, see MODULES_VADDR. |
Ingo Molnar | 88f3aec | 2008-02-21 11:04:11 +0100 | [diff] [blame] | 482 | * If you want to increase this then increase MODULES_VADDR |
| 483 | * too.) |
| 484 | */ |
Jeremy Fitzhardinge | 8490638 | 2008-07-01 16:46:35 -0700 | [diff] [blame] | 485 | PMDS(0, __PAGE_KERNEL_LARGE_EXEC, |
Ingo Molnar | d4afe41 | 2008-02-21 13:39:30 +0100 | [diff] [blame] | 486 | KERNEL_IMAGE_SIZE/PMD_SIZE) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 487 | |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 488 | NEXT_PAGE(level2_fixmap_pgt) |
| 489 | .fill 506,8,0 |
| 490 | .quad level1_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE |
| 491 | /* 8MB reserved for vsyscalls + a 2MB hole = 4 + 1 entries */ |
| 492 | .fill 5,8,0 |
| 493 | |
| 494 | NEXT_PAGE(level1_fixmap_pgt) |
| 495 | .fill 512,8,0 |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 496 | |
Vivek Goyal | 67dcbb6 | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 497 | #undef PMDS |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 498 | |
Jan Beulich | f0cf5d1 | 2006-01-17 07:03:32 +0100 | [diff] [blame] | 499 | .data |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 500 | .align 16 |
Glauber Costa | a939098 | 2008-05-28 16:19:53 -0700 | [diff] [blame] | 501 | .globl early_gdt_descr |
| 502 | early_gdt_descr: |
| 503 | .word GDT_ENTRIES*8-1 |
Tejun Heo | 3e5d8f9 | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 504 | early_gdt_descr_base: |
Brian Gerst | 2add8e2 | 2009-02-08 09:58:39 -0500 | [diff] [blame] | 505 | .quad INIT_PER_CPU_VAR(gdt_page) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 506 | |
Vivek Goyal | 1ab60e0 | 2007-05-02 19:27:07 +0200 | [diff] [blame] | 507 | ENTRY(phys_base) |
| 508 | /* This must match the first entry in level2_kernel_pgt */ |
| 509 | .quad 0x0000000000000000 |
| 510 | |
Jeremy Fitzhardinge | 8c5e5ac | 2008-07-08 15:06:44 -0700 | [diff] [blame] | 511 | #include "../../x86/xen/xen-head.S" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 512 | |
Jan Beulich | e57113b | 2006-03-25 16:30:01 +0100 | [diff] [blame] | 513 | .section .bss, "aw", @nobits |
| 514 | .align L1_CACHE_BYTES |
| 515 | ENTRY(idt_table) |
Cyrill Gorcunov | 5e112ae | 2009-02-23 22:56:58 +0300 | [diff] [blame] | 516 | .skip IDT_ENTRIES * 16 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 517 | |
Steven Rostedt | 228bdaa | 2011-12-09 03:02:19 -0500 | [diff] [blame] | 518 | .align L1_CACHE_BYTES |
| 519 | ENTRY(nmi_idt_table) |
| 520 | .skip IDT_ENTRIES * 16 |
| 521 | |
Tim Abbott | 02b7da3 | 2009-09-20 18:14:14 -0400 | [diff] [blame] | 522 | __PAGE_ALIGNED_BSS |
H. Peter Anvin | 8170e6b | 2013-01-24 12:19:52 -0800 | [diff] [blame] | 523 | NEXT_PAGE(empty_zero_page) |
Jan Beulich | e57113b | 2006-03-25 16:30:01 +0100 | [diff] [blame] | 524 | .skip PAGE_SIZE |