Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1 | /* |
Glauber de Oliveira Costa | 4cedb33 | 2008-03-19 14:26:14 -0300 | [diff] [blame] | 2 | * x86 SMP booting functions |
| 3 | * |
Alan Cox | 87c6fe2 | 2009-01-05 14:08:04 +0000 | [diff] [blame] | 4 | * (c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk> |
Ingo Molnar | 8f47e16 | 2009-01-31 02:03:42 +0100 | [diff] [blame] | 5 | * (c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com> |
Glauber de Oliveira Costa | 4cedb33 | 2008-03-19 14:26:14 -0300 | [diff] [blame] | 6 | * Copyright 2001 Andi Kleen, SuSE Labs. |
| 7 | * |
| 8 | * Much of the core SMP work is based on previous work by Thomas Radke, to |
| 9 | * whom a great many thanks are extended. |
| 10 | * |
| 11 | * Thanks to Intel for making available several different Pentium, |
| 12 | * Pentium Pro and Pentium-II/Xeon MP machines. |
| 13 | * Original development of Linux SMP code supported by Caldera. |
| 14 | * |
| 15 | * This code is released under the GNU General Public License version 2 or |
| 16 | * later. |
| 17 | * |
| 18 | * Fixes |
| 19 | * Felix Koop : NR_CPUS used properly |
| 20 | * Jose Renau : Handle single CPU case. |
| 21 | * Alan Cox : By repeated request 8) - Total BogoMIPS report. |
| 22 | * Greg Wright : Fix for kernel stacks panic. |
| 23 | * Erich Boleyn : MP v1.4 and additional changes. |
| 24 | * Matthias Sattler : Changes for 2.1 kernel map. |
| 25 | * Michel Lespinasse : Changes for 2.1 kernel map. |
| 26 | * Michael Chastain : Change trampoline.S to gnu as. |
| 27 | * Alan Cox : Dumb bug: 'B' step PPro's are fine |
| 28 | * Ingo Molnar : Added APIC timers, based on code |
| 29 | * from Jose Renau |
| 30 | * Ingo Molnar : various cleanups and rewrites |
| 31 | * Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug. |
| 32 | * Maciej W. Rozycki : Bits for genuine 82489DX APICs |
| 33 | * Andi Kleen : Changed for SMP boot into long mode. |
| 34 | * Martin J. Bligh : Added support for multi-quad systems |
| 35 | * Dave Jones : Report invalid combinations of Athlon CPUs. |
| 36 | * Rusty Russell : Hacked into shape for new "hotplug" boot process. |
| 37 | * Andi Kleen : Converted to new state machine. |
| 38 | * Ashok Raj : CPU hotplug support |
| 39 | * Glauber Costa : i386 and x86_64 integration |
| 40 | */ |
| 41 | |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 42 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| 43 | |
Glauber Costa | 68a1c3f | 2008-03-03 14:12:42 -0300 | [diff] [blame] | 44 | #include <linux/init.h> |
| 45 | #include <linux/smp.h> |
Glauber Costa | a355352 | 2008-03-03 14:12:58 -0300 | [diff] [blame] | 46 | #include <linux/module.h> |
Glauber Costa | 70708a1 | 2008-03-03 14:13:03 -0300 | [diff] [blame] | 47 | #include <linux/sched.h> |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 48 | #include <linux/percpu.h> |
Glauber Costa | 91718e8 | 2008-03-03 14:13:12 -0300 | [diff] [blame] | 49 | #include <linux/bootmem.h> |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 50 | #include <linux/err.h> |
| 51 | #include <linux/nmi.h> |
Shane Wang | 69575d3 | 2009-09-01 18:25:07 -0700 | [diff] [blame] | 52 | #include <linux/tboot.h> |
Jacob Pan | 35f720c | 2009-09-17 07:36:43 -0700 | [diff] [blame] | 53 | #include <linux/stackprotector.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 54 | #include <linux/gfp.h> |
Boris Ostrovsky | 1a022e3 | 2012-03-13 19:55:09 +0100 | [diff] [blame] | 55 | #include <linux/cpuidle.h> |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 56 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 57 | #include <asm/acpi.h> |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 58 | #include <asm/desc.h> |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 59 | #include <asm/nmi.h> |
| 60 | #include <asm/irq.h> |
Ingo Molnar | 07bbc16 | 2008-09-23 23:26:42 +0200 | [diff] [blame] | 61 | #include <asm/idle.h> |
Jarkko Sakkinen | 48927bb | 2012-05-08 21:22:28 +0300 | [diff] [blame] | 62 | #include <asm/realmode.h> |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 63 | #include <asm/cpu.h> |
| 64 | #include <asm/numa.h> |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 65 | #include <asm/pgtable.h> |
| 66 | #include <asm/tlbflush.h> |
| 67 | #include <asm/mtrr.h> |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 68 | #include <asm/mwait.h> |
Ingo Molnar | 7b6aa33 | 2009-02-17 13:58:15 +0100 | [diff] [blame] | 69 | #include <asm/apic.h> |
Henrik Kretzschmar | 7167d08 | 2011-02-22 15:38:05 +0100 | [diff] [blame] | 70 | #include <asm/io_apic.h> |
Vincent Palatin | 644c154 | 2012-11-30 12:15:32 -0800 | [diff] [blame] | 71 | #include <asm/i387.h> |
| 72 | #include <asm/fpu-internal.h> |
Yinghai Lu | 569712b | 2008-11-16 03:12:49 -0800 | [diff] [blame] | 73 | #include <asm/setup.h> |
Tejun Heo | bdbcdd4 | 2009-01-21 17:26:06 +0900 | [diff] [blame] | 74 | #include <asm/uv/uv.h> |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 75 | #include <linux/mc146818rtc.h> |
Ingo Molnar | 1164dd0 | 2009-01-28 19:34:09 +0100 | [diff] [blame] | 76 | #include <asm/smpboot_hooks.h> |
Jacob Pan | b81bb37 | 2009-11-09 11:27:04 -0800 | [diff] [blame] | 77 | #include <asm/i8259.h> |
Jarkko Sakkinen | 48927bb | 2012-05-08 21:22:28 +0300 | [diff] [blame] | 78 | #include <asm/realmode.h> |
Borislav Petkov | 646e29a | 2013-09-27 16:35:54 +0200 | [diff] [blame] | 79 | #include <asm/misc.h> |
Jarkko Sakkinen | 48927bb | 2012-05-08 21:22:28 +0300 | [diff] [blame] | 80 | |
Glauber de Oliveira Costa | a8db845 | 2008-03-19 14:26:01 -0300 | [diff] [blame] | 81 | /* State of each CPU */ |
| 82 | DEFINE_PER_CPU(int, cpu_state) = { 0 }; |
| 83 | |
Glauber Costa | a355352 | 2008-03-03 14:12:58 -0300 | [diff] [blame] | 84 | /* Number of siblings per CPU package */ |
| 85 | int smp_num_siblings = 1; |
| 86 | EXPORT_SYMBOL(smp_num_siblings); |
| 87 | |
| 88 | /* Last level cache ID of each logical CPU */ |
Vlad Zolotarov | 0816b0f | 2012-06-11 12:56:52 +0300 | [diff] [blame] | 89 | DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID; |
Glauber Costa | a355352 | 2008-03-03 14:12:58 -0300 | [diff] [blame] | 90 | |
Glauber Costa | a355352 | 2008-03-03 14:12:58 -0300 | [diff] [blame] | 91 | /* representing HT siblings of each logical CPU */ |
Vlad Zolotarov | 0816b0f | 2012-06-11 12:56:52 +0300 | [diff] [blame] | 92 | DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map); |
Glauber Costa | a355352 | 2008-03-03 14:12:58 -0300 | [diff] [blame] | 93 | EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); |
| 94 | |
| 95 | /* representing HT and core siblings of each logical CPU */ |
Vlad Zolotarov | 0816b0f | 2012-06-11 12:56:52 +0300 | [diff] [blame] | 96 | DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map); |
Glauber Costa | a355352 | 2008-03-03 14:12:58 -0300 | [diff] [blame] | 97 | EXPORT_PER_CPU_SYMBOL(cpu_core_map); |
| 98 | |
Vlad Zolotarov | 0816b0f | 2012-06-11 12:56:52 +0300 | [diff] [blame] | 99 | DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map); |
Yinghai Lu | b3d7336 | 2011-01-21 15:29:44 -0800 | [diff] [blame] | 100 | |
Glauber Costa | a355352 | 2008-03-03 14:12:58 -0300 | [diff] [blame] | 101 | /* Per CPU bogomips and other parameters */ |
| 102 | DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info); |
| 103 | EXPORT_PER_CPU_SYMBOL(cpu_info); |
Glauber Costa | 768d950 | 2008-03-03 14:13:02 -0300 | [diff] [blame] | 104 | |
Yinghai Lu | 2b6163b | 2009-02-25 20:50:49 -0800 | [diff] [blame] | 105 | atomic_t init_deasserted; |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 106 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 107 | /* |
Fenghua Yu | 30106c1 | 2012-11-13 11:32:41 -0800 | [diff] [blame] | 108 | * Report back to the Boot Processor during boot time or to the caller processor |
| 109 | * during CPU online. |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 110 | */ |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 111 | static void smp_callin(void) |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 112 | { |
| 113 | int cpuid, phys_id; |
| 114 | unsigned long timeout; |
| 115 | |
| 116 | /* |
| 117 | * If waken up by an INIT in an 82489DX configuration |
| 118 | * we may get here before an INIT-deassert IPI reaches |
| 119 | * our local APIC. We have to wait for the IPI or we'll |
| 120 | * lock up on an APIC access. |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 121 | * |
| 122 | * Since CPU0 is not wakened up by INIT, it doesn't wait for the IPI. |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 123 | */ |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 124 | cpuid = smp_processor_id(); |
| 125 | if (apic->wait_for_init_deassert && cpuid != 0) |
Ingo Molnar | a965936 | 2009-01-28 16:21:32 +0100 | [diff] [blame] | 126 | apic->wait_for_init_deassert(&init_deasserted); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 127 | |
| 128 | /* |
| 129 | * (This works even if the APIC is not enabled.) |
| 130 | */ |
Yinghai Lu | 4c9961d | 2008-07-11 18:44:16 -0700 | [diff] [blame] | 131 | phys_id = read_apic_id(); |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 132 | if (cpumask_test_cpu(cpuid, cpu_callin_mask)) { |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 133 | panic("%s: phys CPU#%d, CPU#%d already present??\n", __func__, |
| 134 | phys_id, cpuid); |
| 135 | } |
Thomas Gleixner | cfc1b9a | 2008-07-21 21:35:38 +0200 | [diff] [blame] | 136 | pr_debug("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 137 | |
| 138 | /* |
| 139 | * STARTUP IPIs are fragile beasts as they might sometimes |
| 140 | * trigger some glue motherboard logic. Complete APIC bus |
| 141 | * silence for 1 second, this overestimates the time the |
| 142 | * boot CPU is spending to send the up to 2 STARTUP IPIs |
| 143 | * by a factor of two. This should be enough. |
| 144 | */ |
| 145 | |
| 146 | /* |
| 147 | * Waiting 2s total for startup (udelay is not yet working) |
| 148 | */ |
| 149 | timeout = jiffies + 2*HZ; |
| 150 | while (time_before(jiffies, timeout)) { |
| 151 | /* |
| 152 | * Has the boot CPU finished it's STARTUP sequence? |
| 153 | */ |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 154 | if (cpumask_test_cpu(cpuid, cpu_callout_mask)) |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 155 | break; |
| 156 | cpu_relax(); |
| 157 | } |
| 158 | |
| 159 | if (!time_before(jiffies, timeout)) { |
| 160 | panic("%s: CPU%d started up but did not get a callout!\n", |
| 161 | __func__, cpuid); |
| 162 | } |
| 163 | |
| 164 | /* |
| 165 | * the boot CPU has finished the init stage and is spinning |
| 166 | * on callin_map until we finish. We are free to set up this |
| 167 | * CPU, first the APIC. (this is probably redundant on most |
| 168 | * boards) |
| 169 | */ |
| 170 | |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 171 | pr_debug("CALLIN, before setup_local_APIC()\n"); |
Ingo Molnar | 333344d | 2009-01-28 16:31:52 +0100 | [diff] [blame] | 172 | if (apic->smp_callin_clear_local_apic) |
| 173 | apic->smp_callin_clear_local_apic(); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 174 | setup_local_APIC(); |
| 175 | end_local_APIC_setup(); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 176 | |
Suresh Siddha | 9d133e5 | 2010-01-29 11:42:21 -0800 | [diff] [blame] | 177 | /* |
| 178 | * Need to setup vector mappings before we enable interrupts. |
| 179 | */ |
Suresh Siddha | 36e9e1e | 2010-03-15 14:33:06 -0800 | [diff] [blame] | 180 | setup_vector_irq(smp_processor_id()); |
Jack Steiner | b565201 | 2011-11-15 15:33:56 -0800 | [diff] [blame] | 181 | |
| 182 | /* |
| 183 | * Save our processor parameters. Note: this information |
| 184 | * is needed for clock calibration. |
| 185 | */ |
| 186 | smp_store_cpu_info(cpuid); |
| 187 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 188 | /* |
| 189 | * Get our bogomips. |
Jack Steiner | b565201 | 2011-11-15 15:33:56 -0800 | [diff] [blame] | 190 | * Update loops_per_jiffy in cpu_data. Previous call to |
| 191 | * smp_store_cpu_info() stored a value that is close but not as |
| 192 | * accurate as the value just calculated. |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 193 | */ |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 194 | calibrate_delay(); |
Jack Steiner | b565201 | 2011-11-15 15:33:56 -0800 | [diff] [blame] | 195 | cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy; |
Thomas Gleixner | cfc1b9a | 2008-07-21 21:35:38 +0200 | [diff] [blame] | 196 | pr_debug("Stack at about %p\n", &cpuid); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 197 | |
| 198 | /* |
Andi Kleen | 5ef428c | 2010-11-18 11:47:31 +0100 | [diff] [blame] | 199 | * This must be done before setting cpu_online_mask |
| 200 | * or calling notify_cpu_starting. |
| 201 | */ |
| 202 | set_cpu_sibling_map(raw_smp_processor_id()); |
| 203 | wmb(); |
| 204 | |
Peter Zijlstra | 8525702 | 2010-03-23 19:30:52 +0100 | [diff] [blame] | 205 | notify_cpu_starting(cpuid); |
| 206 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 207 | /* |
| 208 | * Allow the master to continue. |
| 209 | */ |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 210 | cpumask_set_cpu(cpuid, cpu_callin_mask); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 211 | } |
| 212 | |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 213 | static int cpu0_logical_apicid; |
| 214 | static int enable_start_cpu0; |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 215 | /* |
| 216 | * Activate a secondary processor. |
| 217 | */ |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 218 | static void notrace start_secondary(void *unused) |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 219 | { |
| 220 | /* |
| 221 | * Don't put *anything* before cpu_init(), SMP booting is too |
| 222 | * fragile that we want to limit the things done here to the |
| 223 | * most necessary things. |
| 224 | */ |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 225 | cpu_init(); |
Igor Mammedov | df156f9 | 2012-02-07 15:52:44 +0100 | [diff] [blame] | 226 | x86_cpuinit.early_percpu_clock_init(); |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 227 | preempt_disable(); |
| 228 | smp_callin(); |
| 229 | |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 230 | enable_start_cpu0 = 0; |
| 231 | |
Borislav Petkov | b40827fa | 2010-08-28 15:58:33 +0200 | [diff] [blame] | 232 | #ifdef CONFIG_X86_32 |
| 233 | /* switch away from the initial page table */ |
| 234 | load_cr3(swapper_pg_dir); |
| 235 | __flush_tlb_all(); |
| 236 | #endif |
| 237 | |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 238 | /* otherwise gcc will move up smp_processor_id before the cpu_init */ |
| 239 | barrier(); |
| 240 | /* |
| 241 | * Check TSC synchronization with the BP: |
| 242 | */ |
| 243 | check_tsc_sync_target(); |
| 244 | |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 245 | /* |
Eric W. Biederman | d388e5f | 2008-08-09 15:09:02 -0700 | [diff] [blame] | 246 | * We need to hold vector_lock so there the set of online cpus |
| 247 | * does not change while we are assigning vectors to cpus. Holding |
| 248 | * this lock ensures we don't half assign or remove an irq from a cpu. |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 249 | */ |
Eric W. Biederman | d388e5f | 2008-08-09 15:09:02 -0700 | [diff] [blame] | 250 | lock_vector_lock(); |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 251 | set_cpu_online(smp_processor_id(), true); |
Eric W. Biederman | d388e5f | 2008-08-09 15:09:02 -0700 | [diff] [blame] | 252 | unlock_vector_lock(); |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 253 | per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; |
Russ Anderson | 78c0617 | 2010-02-26 10:49:12 -0600 | [diff] [blame] | 254 | x86_platform.nmi_init(); |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 255 | |
Manfred Spraul | 0cefa5b | 2008-09-07 11:29:58 +0200 | [diff] [blame] | 256 | /* enable local interrupts */ |
| 257 | local_irq_enable(); |
| 258 | |
Jacob Pan | 35f720c | 2009-09-17 07:36:43 -0700 | [diff] [blame] | 259 | /* to prevent fake stack check failure in clock setup */ |
| 260 | boot_init_stack_canary(); |
| 261 | |
Thomas Gleixner | 736deca | 2009-08-19 12:35:53 +0200 | [diff] [blame] | 262 | x86_cpuinit.setup_percpu_clockev(); |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 263 | |
| 264 | wmb(); |
Thomas Gleixner | 7d1a941 | 2013-03-21 22:50:03 +0100 | [diff] [blame] | 265 | cpu_startup_entry(CPUHP_ONLINE); |
Glauber de Oliveira Costa | bbc2ff6 | 2008-03-19 14:26:00 -0300 | [diff] [blame] | 266 | } |
| 267 | |
Fenghua Yu | 30106c1 | 2012-11-13 11:32:41 -0800 | [diff] [blame] | 268 | void __init smp_store_boot_cpu_info(void) |
| 269 | { |
| 270 | int id = 0; /* CPU 0 */ |
| 271 | struct cpuinfo_x86 *c = &cpu_data(id); |
| 272 | |
| 273 | *c = boot_cpu_data; |
| 274 | c->cpu_index = id; |
| 275 | } |
| 276 | |
Glauber de Oliveira Costa | 1d89a7f | 2008-03-19 14:25:05 -0300 | [diff] [blame] | 277 | /* |
| 278 | * The bootstrap kernel entry code has set these up. Save them for |
| 279 | * a given CPU |
| 280 | */ |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 281 | void smp_store_cpu_info(int id) |
Glauber de Oliveira Costa | 1d89a7f | 2008-03-19 14:25:05 -0300 | [diff] [blame] | 282 | { |
| 283 | struct cpuinfo_x86 *c = &cpu_data(id); |
| 284 | |
Yinghai Lu | b3d7336 | 2011-01-21 15:29:44 -0800 | [diff] [blame] | 285 | *c = boot_cpu_data; |
Glauber de Oliveira Costa | 1d89a7f | 2008-03-19 14:25:05 -0300 | [diff] [blame] | 286 | c->cpu_index = id; |
Fenghua Yu | 30106c1 | 2012-11-13 11:32:41 -0800 | [diff] [blame] | 287 | /* |
| 288 | * During boot time, CPU0 has this setup already. Save the info when |
| 289 | * bringing up AP or offlined CPU0. |
| 290 | */ |
| 291 | identify_secondary_cpu(c); |
Glauber de Oliveira Costa | 1d89a7f | 2008-03-19 14:25:05 -0300 | [diff] [blame] | 292 | } |
| 293 | |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 294 | static bool |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 295 | topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name) |
Andreas Herrmann | d4fbe4f | 2010-09-30 14:41:56 +0200 | [diff] [blame] | 296 | { |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 297 | int cpu1 = c->cpu_index, cpu2 = o->cpu_index; |
| 298 | |
| 299 | return !WARN_ONCE(cpu_to_node(cpu1) != cpu_to_node(cpu2), |
| 300 | "sched: CPU #%d's %s-sibling CPU #%d is not on the same node! " |
| 301 | "[node: %d != %d]. Ignoring dependency.\n", |
| 302 | cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2)); |
Andreas Herrmann | d4fbe4f | 2010-09-30 14:41:56 +0200 | [diff] [blame] | 303 | } |
| 304 | |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 305 | #define link_mask(_m, c1, c2) \ |
| 306 | do { \ |
| 307 | cpumask_set_cpu((c1), cpu_##_m##_mask(c2)); \ |
| 308 | cpumask_set_cpu((c2), cpu_##_m##_mask(c1)); \ |
| 309 | } while (0) |
| 310 | |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 311 | static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 312 | { |
Andreas Herrmann | 193f3fc | 2012-10-19 10:58:13 +0200 | [diff] [blame] | 313 | if (cpu_has_topoext) { |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 314 | int cpu1 = c->cpu_index, cpu2 = o->cpu_index; |
| 315 | |
| 316 | if (c->phys_proc_id == o->phys_proc_id && |
| 317 | per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2) && |
| 318 | c->compute_unit_id == o->compute_unit_id) |
| 319 | return topology_sane(c, o, "smt"); |
| 320 | |
| 321 | } else if (c->phys_proc_id == o->phys_proc_id && |
| 322 | c->cpu_core_id == o->cpu_core_id) { |
| 323 | return topology_sane(c, o, "smt"); |
| 324 | } |
| 325 | |
| 326 | return false; |
| 327 | } |
| 328 | |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 329 | static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 330 | { |
| 331 | int cpu1 = c->cpu_index, cpu2 = o->cpu_index; |
| 332 | |
| 333 | if (per_cpu(cpu_llc_id, cpu1) != BAD_APICID && |
| 334 | per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2)) |
| 335 | return topology_sane(c, o, "llc"); |
| 336 | |
| 337 | return false; |
| 338 | } |
| 339 | |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 340 | static bool match_mc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 341 | { |
Borislav Petkov | 161270f | 2012-06-06 17:31:26 +0200 | [diff] [blame] | 342 | if (c->phys_proc_id == o->phys_proc_id) { |
| 343 | if (cpu_has(c, X86_FEATURE_AMD_DCM)) |
| 344 | return true; |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 345 | |
Borislav Petkov | 161270f | 2012-06-06 17:31:26 +0200 | [diff] [blame] | 346 | return topology_sane(c, o, "mc"); |
| 347 | } |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 348 | return false; |
| 349 | } |
Glauber de Oliveira Costa | 1d89a7f | 2008-03-19 14:25:05 -0300 | [diff] [blame] | 350 | |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 351 | void set_cpu_sibling_map(int cpu) |
Glauber Costa | 768d950 | 2008-03-03 14:13:02 -0300 | [diff] [blame] | 352 | { |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 353 | bool has_smt = smp_num_siblings > 1; |
Andrew Jones | b0bc225 | 2013-05-29 14:48:15 +0200 | [diff] [blame] | 354 | bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1; |
Glauber Costa | 768d950 | 2008-03-03 14:13:02 -0300 | [diff] [blame] | 355 | struct cpuinfo_x86 *c = &cpu_data(cpu); |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 356 | struct cpuinfo_x86 *o; |
| 357 | int i; |
Glauber Costa | 768d950 | 2008-03-03 14:13:02 -0300 | [diff] [blame] | 358 | |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 359 | cpumask_set_cpu(cpu, cpu_sibling_setup_mask); |
Glauber Costa | 768d950 | 2008-03-03 14:13:02 -0300 | [diff] [blame] | 360 | |
Andrew Jones | b0bc225 | 2013-05-29 14:48:15 +0200 | [diff] [blame] | 361 | if (!has_mp) { |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 362 | cpumask_set_cpu(cpu, cpu_sibling_mask(cpu)); |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 363 | cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu)); |
| 364 | cpumask_set_cpu(cpu, cpu_core_mask(cpu)); |
Glauber Costa | 768d950 | 2008-03-03 14:13:02 -0300 | [diff] [blame] | 365 | c->booted_cores = 1; |
| 366 | return; |
| 367 | } |
| 368 | |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 369 | for_each_cpu(i, cpu_sibling_setup_mask) { |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 370 | o = &cpu_data(i); |
Peter Zijlstra | 0acbb44 | 2012-04-18 19:04:09 +0200 | [diff] [blame] | 371 | |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 372 | if ((i == cpu) || (has_smt && match_smt(c, o))) |
| 373 | link_mask(sibling, cpu, i); |
Peter Zijlstra | 0acbb44 | 2012-04-18 19:04:09 +0200 | [diff] [blame] | 374 | |
Andrew Jones | b0bc225 | 2013-05-29 14:48:15 +0200 | [diff] [blame] | 375 | if ((i == cpu) || (has_mp && match_llc(c, o))) |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 376 | link_mask(llc_shared, cpu, i); |
| 377 | |
Kamalesh Babulal | ceb1cba | 2012-05-31 13:07:38 +0530 | [diff] [blame] | 378 | } |
| 379 | |
| 380 | /* |
| 381 | * This needs a separate iteration over the cpus because we rely on all |
| 382 | * cpu_sibling_mask links to be set-up. |
| 383 | */ |
| 384 | for_each_cpu(i, cpu_sibling_setup_mask) { |
| 385 | o = &cpu_data(i); |
| 386 | |
Andrew Jones | b0bc225 | 2013-05-29 14:48:15 +0200 | [diff] [blame] | 387 | if ((i == cpu) || (has_mp && match_mc(c, o))) { |
Peter Zijlstra | 316ad24 | 2012-05-11 13:05:59 +0200 | [diff] [blame] | 388 | link_mask(core, cpu, i); |
| 389 | |
Glauber Costa | 768d950 | 2008-03-03 14:13:02 -0300 | [diff] [blame] | 390 | /* |
| 391 | * Does this new cpu bringup a new core? |
| 392 | */ |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 393 | if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) { |
Glauber Costa | 768d950 | 2008-03-03 14:13:02 -0300 | [diff] [blame] | 394 | /* |
| 395 | * for each core in package, increment |
| 396 | * the booted_cores for this new cpu |
| 397 | */ |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 398 | if (cpumask_first(cpu_sibling_mask(i)) == i) |
Glauber Costa | 768d950 | 2008-03-03 14:13:02 -0300 | [diff] [blame] | 399 | c->booted_cores++; |
| 400 | /* |
| 401 | * increment the core count for all |
| 402 | * the other cpus in this package |
| 403 | */ |
| 404 | if (i != cpu) |
| 405 | cpu_data(i).booted_cores++; |
| 406 | } else if (i != cpu && !c->booted_cores) |
| 407 | c->booted_cores = cpu_data(i).booted_cores; |
| 408 | } |
| 409 | } |
| 410 | } |
| 411 | |
Glauber Costa | 70708a1 | 2008-03-03 14:13:03 -0300 | [diff] [blame] | 412 | /* maps the cpu to the sched domain representing multi-core */ |
Rusty Russell | 030bb20 | 2008-12-26 22:23:41 +1030 | [diff] [blame] | 413 | const struct cpumask *cpu_coregroup_mask(int cpu) |
Glauber Costa | 70708a1 | 2008-03-03 14:13:03 -0300 | [diff] [blame] | 414 | { |
Peter Zijlstra | 9f64638 | 2012-05-29 16:39:09 +0200 | [diff] [blame] | 415 | return cpu_llc_shared_mask(cpu); |
Rusty Russell | 030bb20 | 2008-12-26 22:23:41 +1030 | [diff] [blame] | 416 | } |
| 417 | |
Ingo Molnar | a4928cf | 2008-04-23 13:20:56 +0200 | [diff] [blame] | 418 | static void impress_friends(void) |
Glauber de Oliveira Costa | 904541e | 2008-03-19 14:25:27 -0300 | [diff] [blame] | 419 | { |
| 420 | int cpu; |
| 421 | unsigned long bogosum = 0; |
| 422 | /* |
| 423 | * Allow the user to impress friends. |
| 424 | */ |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 425 | pr_debug("Before bogomips\n"); |
Glauber de Oliveira Costa | 904541e | 2008-03-19 14:25:27 -0300 | [diff] [blame] | 426 | for_each_possible_cpu(cpu) |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 427 | if (cpumask_test_cpu(cpu, cpu_callout_mask)) |
Glauber de Oliveira Costa | 904541e | 2008-03-19 14:25:27 -0300 | [diff] [blame] | 428 | bogosum += cpu_data(cpu).loops_per_jiffy; |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 429 | pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n", |
Glauber de Oliveira Costa | f68e00a | 2008-03-19 14:25:29 -0300 | [diff] [blame] | 430 | num_online_cpus(), |
Glauber de Oliveira Costa | 904541e | 2008-03-19 14:25:27 -0300 | [diff] [blame] | 431 | bogosum/(500000/HZ), |
| 432 | (bogosum/(5000/HZ))%100); |
| 433 | |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 434 | pr_debug("Before bogocount - setting activated=1\n"); |
Glauber de Oliveira Costa | 904541e | 2008-03-19 14:25:27 -0300 | [diff] [blame] | 435 | } |
| 436 | |
Yinghai Lu | 569712b | 2008-11-16 03:12:49 -0800 | [diff] [blame] | 437 | void __inquire_remote_apic(int apicid) |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 438 | { |
| 439 | unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 }; |
Greg Dietsche | a6c23905 | 2011-06-30 20:10:53 -0500 | [diff] [blame] | 440 | const char * const names[] = { "ID", "VERSION", "SPIV" }; |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 441 | int timeout; |
| 442 | u32 status; |
| 443 | |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 444 | pr_info("Inquiring remote APIC 0x%x...\n", apicid); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 445 | |
| 446 | for (i = 0; i < ARRAY_SIZE(regs); i++) { |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 447 | pr_info("... APIC 0x%x %s: ", apicid, names[i]); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 448 | |
| 449 | /* |
| 450 | * Wait for idle. |
| 451 | */ |
| 452 | status = safe_apic_wait_icr_idle(); |
| 453 | if (status) |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 454 | pr_cont("a previous APIC delivery may have failed\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 455 | |
Suresh Siddha | 1b374e4 | 2008-07-10 11:16:49 -0700 | [diff] [blame] | 456 | apic_icr_write(APIC_DM_REMRD | regs[i], apicid); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 457 | |
| 458 | timeout = 0; |
| 459 | do { |
| 460 | udelay(100); |
| 461 | status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK; |
| 462 | } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000); |
| 463 | |
| 464 | switch (status) { |
| 465 | case APIC_ICR_RR_VALID: |
| 466 | status = apic_read(APIC_RRR); |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 467 | pr_cont("%08x\n", status); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 468 | break; |
| 469 | default: |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 470 | pr_cont("failed\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 471 | } |
| 472 | } |
| 473 | } |
| 474 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 475 | /* |
| 476 | * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal |
| 477 | * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this |
| 478 | * won't ... remember to clear down the APIC, etc later. |
| 479 | */ |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 480 | int |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 481 | wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip) |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 482 | { |
| 483 | unsigned long send_status, accept_status = 0; |
| 484 | int maxlvt; |
| 485 | |
| 486 | /* Target chip */ |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 487 | /* Boot on the stack */ |
| 488 | /* Kick the second */ |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 489 | apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 490 | |
Thomas Gleixner | cfc1b9a | 2008-07-21 21:35:38 +0200 | [diff] [blame] | 491 | pr_debug("Waiting for send to finish...\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 492 | send_status = safe_apic_wait_icr_idle(); |
| 493 | |
| 494 | /* |
| 495 | * Give the other CPU some time to accept the IPI. |
| 496 | */ |
| 497 | udelay(200); |
Yinghai Lu | 569712b | 2008-11-16 03:12:49 -0800 | [diff] [blame] | 498 | if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) { |
Cyrill Gorcunov | 59ef48a | 2008-09-14 21:58:49 +0400 | [diff] [blame] | 499 | maxlvt = lapic_get_maxlvt(); |
| 500 | if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ |
| 501 | apic_write(APIC_ESR, 0); |
| 502 | accept_status = (apic_read(APIC_ESR) & 0xEF); |
| 503 | } |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 504 | pr_debug("NMI sent\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 505 | |
| 506 | if (send_status) |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 507 | pr_err("APIC never delivered???\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 508 | if (accept_status) |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 509 | pr_err("APIC delivery error (%lx)\n", accept_status); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 510 | |
| 511 | return (send_status | accept_status); |
| 512 | } |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 513 | |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 514 | static int |
Yinghai Lu | 569712b | 2008-11-16 03:12:49 -0800 | [diff] [blame] | 515 | wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip) |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 516 | { |
| 517 | unsigned long send_status, accept_status = 0; |
| 518 | int maxlvt, num_starts, j; |
| 519 | |
Maciej W. Rozycki | 593f4a7 | 2008-07-16 19:15:30 +0100 | [diff] [blame] | 520 | maxlvt = lapic_get_maxlvt(); |
| 521 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 522 | /* |
| 523 | * Be paranoid about clearing APIC errors. |
| 524 | */ |
| 525 | if (APIC_INTEGRATED(apic_version[phys_apicid])) { |
Maciej W. Rozycki | 593f4a7 | 2008-07-16 19:15:30 +0100 | [diff] [blame] | 526 | if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ |
| 527 | apic_write(APIC_ESR, 0); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 528 | apic_read(APIC_ESR); |
| 529 | } |
| 530 | |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 531 | pr_debug("Asserting INIT\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 532 | |
| 533 | /* |
| 534 | * Turn INIT on target chip |
| 535 | */ |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 536 | /* |
| 537 | * Send IPI |
| 538 | */ |
Suresh Siddha | 1b374e4 | 2008-07-10 11:16:49 -0700 | [diff] [blame] | 539 | apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT, |
| 540 | phys_apicid); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 541 | |
Thomas Gleixner | cfc1b9a | 2008-07-21 21:35:38 +0200 | [diff] [blame] | 542 | pr_debug("Waiting for send to finish...\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 543 | send_status = safe_apic_wait_icr_idle(); |
| 544 | |
| 545 | mdelay(10); |
| 546 | |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 547 | pr_debug("Deasserting INIT\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 548 | |
| 549 | /* Target chip */ |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 550 | /* Send IPI */ |
Suresh Siddha | 1b374e4 | 2008-07-10 11:16:49 -0700 | [diff] [blame] | 551 | apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 552 | |
Thomas Gleixner | cfc1b9a | 2008-07-21 21:35:38 +0200 | [diff] [blame] | 553 | pr_debug("Waiting for send to finish...\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 554 | send_status = safe_apic_wait_icr_idle(); |
| 555 | |
| 556 | mb(); |
| 557 | atomic_set(&init_deasserted, 1); |
| 558 | |
| 559 | /* |
| 560 | * Should we send STARTUP IPIs ? |
| 561 | * |
| 562 | * Determine this based on the APIC version. |
| 563 | * If we don't have an integrated APIC, don't send the STARTUP IPIs. |
| 564 | */ |
| 565 | if (APIC_INTEGRATED(apic_version[phys_apicid])) |
| 566 | num_starts = 2; |
| 567 | else |
| 568 | num_starts = 0; |
| 569 | |
| 570 | /* |
| 571 | * Paravirt / VMI wants a startup IPI hook here to set up the |
| 572 | * target processor state. |
| 573 | */ |
| 574 | startup_ipi_hook(phys_apicid, (unsigned long) start_secondary, |
H. Peter Anvin | 11d4c3f | 2011-02-04 16:14:11 -0800 | [diff] [blame] | 575 | stack_start); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 576 | |
| 577 | /* |
| 578 | * Run STARTUP IPI loop. |
| 579 | */ |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 580 | pr_debug("#startup loops: %d\n", num_starts); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 581 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 582 | for (j = 1; j <= num_starts; j++) { |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 583 | pr_debug("Sending STARTUP #%d\n", j); |
Maciej W. Rozycki | 593f4a7 | 2008-07-16 19:15:30 +0100 | [diff] [blame] | 584 | if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ |
| 585 | apic_write(APIC_ESR, 0); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 586 | apic_read(APIC_ESR); |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 587 | pr_debug("After apic_write\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 588 | |
| 589 | /* |
| 590 | * STARTUP IPI |
| 591 | */ |
| 592 | |
| 593 | /* Target chip */ |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 594 | /* Boot on the stack */ |
| 595 | /* Kick the second */ |
Suresh Siddha | 1b374e4 | 2008-07-10 11:16:49 -0700 | [diff] [blame] | 596 | apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12), |
| 597 | phys_apicid); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 598 | |
| 599 | /* |
| 600 | * Give the other CPU some time to accept the IPI. |
| 601 | */ |
| 602 | udelay(300); |
| 603 | |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 604 | pr_debug("Startup point 1\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 605 | |
Thomas Gleixner | cfc1b9a | 2008-07-21 21:35:38 +0200 | [diff] [blame] | 606 | pr_debug("Waiting for send to finish...\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 607 | send_status = safe_apic_wait_icr_idle(); |
| 608 | |
| 609 | /* |
| 610 | * Give the other CPU some time to accept the IPI. |
| 611 | */ |
| 612 | udelay(200); |
Maciej W. Rozycki | 593f4a7 | 2008-07-16 19:15:30 +0100 | [diff] [blame] | 613 | if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 614 | apic_write(APIC_ESR, 0); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 615 | accept_status = (apic_read(APIC_ESR) & 0xEF); |
| 616 | if (send_status || accept_status) |
| 617 | break; |
| 618 | } |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 619 | pr_debug("After Startup\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 620 | |
| 621 | if (send_status) |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 622 | pr_err("APIC never delivered???\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 623 | if (accept_status) |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 624 | pr_err("APIC delivery error (%lx)\n", accept_status); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 625 | |
| 626 | return (send_status | accept_status); |
| 627 | } |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 628 | |
Borislav Petkov | a17bce4 | 2013-09-30 11:56:24 +0200 | [diff] [blame] | 629 | void smp_announce(void) |
| 630 | { |
| 631 | int num_nodes = num_online_nodes(); |
| 632 | |
| 633 | printk(KERN_INFO "x86: Booted up %d node%s, %d CPUs\n", |
| 634 | num_nodes, (num_nodes > 1 ? "s" : ""), num_online_cpus()); |
| 635 | } |
| 636 | |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 637 | /* reduce the number of lines printed when booting a large cpu count system */ |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 638 | static void announce_cpu(int cpu, int apicid) |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 639 | { |
| 640 | static int current_node = -1; |
Borislav Petkov | 4adc8b7 | 2010-06-01 21:04:55 +0200 | [diff] [blame] | 641 | int node = early_cpu_to_node(cpu); |
Borislav Petkov | a17bce4 | 2013-09-30 11:56:24 +0200 | [diff] [blame] | 642 | static int width, node_width; |
Borislav Petkov | 646e29a | 2013-09-27 16:35:54 +0200 | [diff] [blame] | 643 | |
| 644 | if (!width) |
| 645 | width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */ |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 646 | |
Borislav Petkov | a17bce4 | 2013-09-30 11:56:24 +0200 | [diff] [blame] | 647 | if (!node_width) |
| 648 | node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */ |
| 649 | |
| 650 | if (cpu == 1) |
| 651 | printk(KERN_INFO "x86: Booting SMP configuration:\n"); |
| 652 | |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 653 | if (system_state == SYSTEM_BOOTING) { |
| 654 | if (node != current_node) { |
| 655 | if (current_node > (-1)) |
Borislav Petkov | a17bce4 | 2013-09-30 11:56:24 +0200 | [diff] [blame] | 656 | pr_cont("\n"); |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 657 | current_node = node; |
Borislav Petkov | a17bce4 | 2013-09-30 11:56:24 +0200 | [diff] [blame] | 658 | |
| 659 | printk(KERN_INFO ".... node %*s#%d, CPUs: ", |
| 660 | node_width - num_digits(node), " ", node); |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 661 | } |
Borislav Petkov | 646e29a | 2013-09-27 16:35:54 +0200 | [diff] [blame] | 662 | |
| 663 | /* Add padding for the BSP */ |
| 664 | if (cpu == 1) |
| 665 | pr_cont("%*s", width + 1, " "); |
| 666 | |
| 667 | pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu); |
| 668 | |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 669 | } else |
| 670 | pr_info("Booting Node %d Processor %d APIC 0x%x\n", |
| 671 | node, cpu, apicid); |
| 672 | } |
| 673 | |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 674 | static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs) |
| 675 | { |
| 676 | int cpu; |
| 677 | |
| 678 | cpu = smp_processor_id(); |
| 679 | if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0) |
| 680 | return NMI_HANDLED; |
| 681 | |
| 682 | return NMI_DONE; |
| 683 | } |
| 684 | |
| 685 | /* |
| 686 | * Wake up AP by INIT, INIT, STARTUP sequence. |
| 687 | * |
| 688 | * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS |
| 689 | * boot-strap code which is not a desired behavior for waking up BSP. To |
| 690 | * void the boot-strap code, wake up CPU0 by NMI instead. |
| 691 | * |
| 692 | * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined |
| 693 | * (i.e. physically hot removed and then hot added), NMI won't wake it up. |
| 694 | * We'll change this code in the future to wake up hard offlined CPU0 if |
| 695 | * real platform and request are available. |
| 696 | */ |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 697 | static int |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 698 | wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid, |
| 699 | int *cpu0_nmi_registered) |
| 700 | { |
| 701 | int id; |
| 702 | int boot_error; |
| 703 | |
| 704 | /* |
| 705 | * Wake up AP by INIT, INIT, STARTUP sequence. |
| 706 | */ |
| 707 | if (cpu) |
| 708 | return wakeup_secondary_cpu_via_init(apicid, start_ip); |
| 709 | |
| 710 | /* |
| 711 | * Wake up BSP by nmi. |
| 712 | * |
| 713 | * Register a NMI handler to help wake up CPU0. |
| 714 | */ |
| 715 | boot_error = register_nmi_handler(NMI_LOCAL, |
| 716 | wakeup_cpu0_nmi, 0, "wake_cpu0"); |
| 717 | |
| 718 | if (!boot_error) { |
| 719 | enable_start_cpu0 = 1; |
| 720 | *cpu0_nmi_registered = 1; |
| 721 | if (apic->dest_logical == APIC_DEST_LOGICAL) |
| 722 | id = cpu0_logical_apicid; |
| 723 | else |
| 724 | id = apicid; |
| 725 | boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip); |
| 726 | } |
| 727 | |
| 728 | return boot_error; |
| 729 | } |
| 730 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 731 | /* |
| 732 | * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad |
| 733 | * (ie clustered apic addressing mode), this is a LOGICAL apic ID. |
Ingo Molnar | 1f5bcab | 2009-02-26 13:51:40 +0100 | [diff] [blame] | 734 | * Returns zero if CPU booted OK, else error code from |
| 735 | * ->wakeup_secondary_cpu. |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 736 | */ |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 737 | static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle) |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 738 | { |
Jarkko Sakkinen | 48927bb | 2012-05-08 21:22:28 +0300 | [diff] [blame] | 739 | volatile u32 *trampoline_status = |
Jarkko Sakkinen | b429dbf | 2012-05-08 21:22:41 +0300 | [diff] [blame] | 740 | (volatile u32 *) __va(real_mode_header->trampoline_status); |
Jarkko Sakkinen | 48927bb | 2012-05-08 21:22:28 +0300 | [diff] [blame] | 741 | /* start_ip had better be page-aligned! */ |
Jarkko Sakkinen | f37240f | 2012-05-08 21:22:43 +0300 | [diff] [blame] | 742 | unsigned long start_ip = real_mode_header->trampoline_start; |
Jarkko Sakkinen | 48927bb | 2012-05-08 21:22:28 +0300 | [diff] [blame] | 743 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 744 | unsigned long boot_error = 0; |
Ingo Molnar | ab6fb7c | 2009-02-17 16:22:09 +0100 | [diff] [blame] | 745 | int timeout; |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 746 | int cpu0_nmi_registered = 0; |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 747 | |
Rusty Russell | 816afe4 | 2012-08-06 17:29:49 +0930 | [diff] [blame] | 748 | /* Just in case we booted with a single CPU. */ |
| 749 | alternatives_enable_smp(); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 750 | |
Thomas Gleixner | 7eb43a6 | 2012-04-20 13:05:48 +0000 | [diff] [blame] | 751 | idle->thread.sp = (unsigned long) (((struct pt_regs *) |
| 752 | (THREAD_SIZE + task_stack_page(idle))) - 1); |
| 753 | per_cpu(current_task, cpu) = idle; |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 754 | |
Brian Gerst | c6f5e0a | 2009-01-19 00:38:58 +0900 | [diff] [blame] | 755 | #ifdef CONFIG_X86_32 |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 756 | /* Stack for startup_32 can be just as for start_secondary onwards */ |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 757 | irq_ctx_init(cpu); |
| 758 | #else |
Thomas Gleixner | 7eb43a6 | 2012-04-20 13:05:48 +0000 | [diff] [blame] | 759 | clear_tsk_thread_flag(idle, TIF_FORK); |
Tejun Heo | 004aa32 | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 760 | initial_gs = per_cpu_offset(cpu); |
Brian Gerst | 9af4565 | 2009-01-19 00:38:58 +0900 | [diff] [blame] | 761 | per_cpu(kernel_stack, cpu) = |
Thomas Gleixner | 7eb43a6 | 2012-04-20 13:05:48 +0000 | [diff] [blame] | 762 | (unsigned long)task_stack_page(idle) - |
Brian Gerst | 9af4565 | 2009-01-19 00:38:58 +0900 | [diff] [blame] | 763 | KERNEL_STACK_OFFSET + THREAD_SIZE; |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 764 | #endif |
Glauber Costa | a939098 | 2008-05-28 16:19:53 -0700 | [diff] [blame] | 765 | early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu); |
Glauber Costa | 3e97047 | 2008-05-28 13:01:54 -0300 | [diff] [blame] | 766 | initial_code = (unsigned long)start_secondary; |
Thomas Gleixner | 7eb43a6 | 2012-04-20 13:05:48 +0000 | [diff] [blame] | 767 | stack_start = idle->thread.sp; |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 768 | |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 769 | /* So we see what's up */ |
| 770 | announce_cpu(cpu, apicid); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 771 | |
| 772 | /* |
| 773 | * This grunge runs the startup process for |
| 774 | * the targeted processor. |
| 775 | */ |
| 776 | |
| 777 | atomic_set(&init_deasserted, 0); |
| 778 | |
Jack Steiner | 34d0559 | 2008-04-16 11:45:15 -0500 | [diff] [blame] | 779 | if (get_uv_system_type() != UV_NON_UNIQUE_APIC) { |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 780 | |
Thomas Gleixner | cfc1b9a | 2008-07-21 21:35:38 +0200 | [diff] [blame] | 781 | pr_debug("Setting warm reset code and vector.\n"); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 782 | |
Jack Steiner | 34d0559 | 2008-04-16 11:45:15 -0500 | [diff] [blame] | 783 | smpboot_setup_warm_reset_vector(start_ip); |
| 784 | /* |
| 785 | * Be paranoid about clearing APIC errors. |
Cyrill Gorcunov | db96b0a | 2008-10-22 18:00:09 +0400 | [diff] [blame] | 786 | */ |
| 787 | if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) { |
| 788 | apic_write(APIC_ESR, 0); |
| 789 | apic_read(APIC_ESR); |
| 790 | } |
Jack Steiner | 34d0559 | 2008-04-16 11:45:15 -0500 | [diff] [blame] | 791 | } |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 792 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 793 | /* |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 794 | * Wake up a CPU in difference cases: |
| 795 | * - Use the method in the APIC driver if it's defined |
| 796 | * Otherwise, |
| 797 | * - Use an INIT boot APIC message for APs or NMI for BSP. |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 798 | */ |
Ingo Molnar | 1f5bcab | 2009-02-26 13:51:40 +0100 | [diff] [blame] | 799 | if (apic->wakeup_secondary_cpu) |
| 800 | boot_error = apic->wakeup_secondary_cpu(apicid, start_ip); |
| 801 | else |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 802 | boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid, |
| 803 | &cpu0_nmi_registered); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 804 | |
| 805 | if (!boot_error) { |
| 806 | /* |
| 807 | * allow APs to start initializing. |
| 808 | */ |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 809 | pr_debug("Before Callout %d\n", cpu); |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 810 | cpumask_set_cpu(cpu, cpu_callout_mask); |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 811 | pr_debug("After Callout %d\n", cpu); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 812 | |
| 813 | /* |
| 814 | * Wait 5s total for a response |
| 815 | */ |
| 816 | for (timeout = 0; timeout < 50000; timeout++) { |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 817 | if (cpumask_test_cpu(cpu, cpu_callin_mask)) |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 818 | break; /* It has booted */ |
| 819 | udelay(100); |
Suresh Siddha | 68f202e | 2010-07-30 11:46:42 -0700 | [diff] [blame] | 820 | /* |
| 821 | * Allow other tasks to run while we wait for the |
| 822 | * AP to come online. This also gives a chance |
| 823 | * for the MTRR work(triggered by the AP coming online) |
| 824 | * to be completed in the stop machine context. |
| 825 | */ |
| 826 | schedule(); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 827 | } |
| 828 | |
Yinghai Lu | 21c3fcf | 2012-02-12 09:53:57 -0800 | [diff] [blame] | 829 | if (cpumask_test_cpu(cpu, cpu_callin_mask)) { |
| 830 | print_cpu_msr(&cpu_data(cpu)); |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 831 | pr_debug("CPU%d: has booted.\n", cpu); |
Yinghai Lu | 21c3fcf | 2012-02-12 09:53:57 -0800 | [diff] [blame] | 832 | } else { |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 833 | boot_error = 1; |
Jarkko Sakkinen | 48927bb | 2012-05-08 21:22:28 +0300 | [diff] [blame] | 834 | if (*trampoline_status == 0xA5A5A5A5) |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 835 | /* trampoline started but...? */ |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 836 | pr_err("CPU%d: Stuck ??\n", cpu); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 837 | else |
| 838 | /* trampoline code not run */ |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 839 | pr_err("CPU%d: Not responding\n", cpu); |
Ingo Molnar | 25dc004 | 2009-01-28 16:31:52 +0100 | [diff] [blame] | 840 | if (apic->inquire_remote_apic) |
| 841 | apic->inquire_remote_apic(apicid); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 842 | } |
| 843 | } |
Tejun Heo | 1a51e3a | 2009-01-13 20:41:35 +0900 | [diff] [blame] | 844 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 845 | if (boot_error) { |
| 846 | /* Try to put things back the way they were before ... */ |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 847 | numa_remove_cpu(cpu); /* was set by numa_add_cpu */ |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 848 | |
| 849 | /* was set by do_boot_cpu() */ |
| 850 | cpumask_clear_cpu(cpu, cpu_callout_mask); |
| 851 | |
| 852 | /* was set by cpu_init() */ |
| 853 | cpumask_clear_cpu(cpu, cpu_initialized_mask); |
| 854 | |
| 855 | set_cpu_present(cpu, false); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 856 | per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID; |
| 857 | } |
| 858 | |
| 859 | /* mark "stuck" area as not stuck */ |
Jarkko Sakkinen | 48927bb | 2012-05-08 21:22:28 +0300 | [diff] [blame] | 860 | *trampoline_status = 0; |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 861 | |
Yinghai Lu | 02421f9 | 2009-04-03 17:15:53 -0700 | [diff] [blame] | 862 | if (get_uv_system_type() != UV_NON_UNIQUE_APIC) { |
| 863 | /* |
| 864 | * Cleanup possible dangling ends... |
| 865 | */ |
| 866 | smpboot_restore_warm_reset_vector(); |
| 867 | } |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 868 | /* |
| 869 | * Clean up the nmi handler. Do this after the callin and callout sync |
| 870 | * to avoid impact of possible long unregister time. |
| 871 | */ |
| 872 | if (cpu0_nmi_registered) |
| 873 | unregister_nmi_handler(NMI_LOCAL, "wake_cpu0"); |
| 874 | |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 875 | return boot_error; |
| 876 | } |
| 877 | |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 878 | int native_cpu_up(unsigned int cpu, struct task_struct *tidle) |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 879 | { |
Ingo Molnar | a21769a4 | 2009-01-28 06:50:47 +0100 | [diff] [blame] | 880 | int apicid = apic->cpu_present_to_apicid(cpu); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 881 | unsigned long flags; |
| 882 | int err; |
| 883 | |
| 884 | WARN_ON(irqs_disabled()); |
| 885 | |
Thomas Gleixner | cfc1b9a | 2008-07-21 21:35:38 +0200 | [diff] [blame] | 886 | pr_debug("++++++++++++++++++++=_---CPU UP %u\n", cpu); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 887 | |
Fenghua Yu | 30106c1 | 2012-11-13 11:32:41 -0800 | [diff] [blame] | 888 | if (apicid == BAD_APICID || |
Suresh Siddha | c284b42 | 2011-12-21 17:45:19 -0800 | [diff] [blame] | 889 | !physid_isset(apicid, phys_cpu_present_map) || |
Daniel J Blueman | fa63030 | 2012-03-14 15:17:34 +0800 | [diff] [blame] | 890 | !apic->apic_id_valid(apicid)) { |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 891 | pr_err("%s: bad cpu %d\n", __func__, cpu); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 892 | return -EINVAL; |
| 893 | } |
| 894 | |
| 895 | /* |
| 896 | * Already booted CPU? |
| 897 | */ |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 898 | if (cpumask_test_cpu(cpu, cpu_callin_mask)) { |
Thomas Gleixner | cfc1b9a | 2008-07-21 21:35:38 +0200 | [diff] [blame] | 899 | pr_debug("do_boot_cpu %d Already started\n", cpu); |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 900 | return -ENOSYS; |
| 901 | } |
| 902 | |
| 903 | /* |
| 904 | * Save current MTRR state in case it was changed since early boot |
| 905 | * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync: |
| 906 | */ |
| 907 | mtrr_save_state(); |
| 908 | |
| 909 | per_cpu(cpu_state, cpu) = CPU_UP_PREPARE; |
| 910 | |
Vincent Palatin | 644c154 | 2012-11-30 12:15:32 -0800 | [diff] [blame] | 911 | /* the FPU context is blank, nobody can own it */ |
| 912 | __cpu_disable_lazy_restore(cpu); |
| 913 | |
Thomas Gleixner | 7eb43a6 | 2012-04-20 13:05:48 +0000 | [diff] [blame] | 914 | err = do_boot_cpu(apicid, cpu, tidle); |
Hugh Dickins | 61165d7 | 2008-05-13 14:26:57 +0100 | [diff] [blame] | 915 | if (err) { |
Thomas Gleixner | cfc1b9a | 2008-07-21 21:35:38 +0200 | [diff] [blame] | 916 | pr_debug("do_boot_cpu failed %d\n", err); |
Hugh Dickins | 61165d7 | 2008-05-13 14:26:57 +0100 | [diff] [blame] | 917 | return -EIO; |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 918 | } |
| 919 | |
| 920 | /* |
| 921 | * Check TSC synchronization with the AP (keep irqs disabled |
| 922 | * while doing so): |
| 923 | */ |
| 924 | local_irq_save(flags); |
| 925 | check_tsc_sync_source(cpu); |
| 926 | local_irq_restore(flags); |
| 927 | |
Akinobu Mita | 7c04e64 | 2008-04-19 23:55:17 +0900 | [diff] [blame] | 928 | while (!cpu_online(cpu)) { |
Glauber de Oliveira Costa | cb3c8b9 | 2008-03-19 14:25:59 -0300 | [diff] [blame] | 929 | cpu_relax(); |
| 930 | touch_nmi_watchdog(); |
| 931 | } |
| 932 | |
| 933 | return 0; |
| 934 | } |
| 935 | |
Henrik Kretzschmar | 7167d08 | 2011-02-22 15:38:05 +0100 | [diff] [blame] | 936 | /** |
| 937 | * arch_disable_smp_support() - disables SMP support for x86 at runtime |
| 938 | */ |
| 939 | void arch_disable_smp_support(void) |
| 940 | { |
| 941 | disable_ioapic_support(); |
| 942 | } |
| 943 | |
Glauber de Oliveira Costa | a8db845 | 2008-03-19 14:26:01 -0300 | [diff] [blame] | 944 | /* |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 945 | * Fall back to non SMP mode after errors. |
| 946 | * |
| 947 | * RED-PEN audit/test this more. I bet there is more state messed up here. |
| 948 | */ |
| 949 | static __init void disable_smp(void) |
| 950 | { |
Rusty Russell | 4f06289 | 2009-03-13 14:49:54 +1030 | [diff] [blame] | 951 | init_cpu_present(cpumask_of(0)); |
| 952 | init_cpu_possible(cpumask_of(0)); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 953 | smpboot_clear_io_apic_irqs(); |
Glauber Costa | 0f385d1 | 2008-05-28 17:09:53 -0700 | [diff] [blame] | 954 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 955 | if (smp_found_config) |
Jack Steiner | b6df1b8 | 2008-06-19 21:51:05 -0500 | [diff] [blame] | 956 | physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 957 | else |
Jack Steiner | b6df1b8 | 2008-06-19 21:51:05 -0500 | [diff] [blame] | 958 | physid_set_mask_of_physid(0, &phys_cpu_present_map); |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 959 | cpumask_set_cpu(0, cpu_sibling_mask(0)); |
| 960 | cpumask_set_cpu(0, cpu_core_mask(0)); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 961 | } |
| 962 | |
| 963 | /* |
| 964 | * Various sanity checks. |
| 965 | */ |
| 966 | static int __init smp_sanity_check(unsigned max_cpus) |
| 967 | { |
Jack Steiner | ac23d4e | 2008-03-28 14:12:16 -0500 | [diff] [blame] | 968 | preempt_disable(); |
Yinghai Lu | a58f03b | 2008-08-14 02:16:30 -0700 | [diff] [blame] | 969 | |
Yinghai Lu | 1ff2f20 | 2009-01-29 19:30:04 -0800 | [diff] [blame] | 970 | #if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32) |
Yinghai Lu | a58f03b | 2008-08-14 02:16:30 -0700 | [diff] [blame] | 971 | if (def_to_bigsmp && nr_cpu_ids > 8) { |
| 972 | unsigned int cpu; |
| 973 | unsigned nr; |
| 974 | |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 975 | pr_warn("More than 8 CPUs detected - skipping them\n" |
| 976 | "Use CONFIG_X86_BIGSMP\n"); |
Yinghai Lu | a58f03b | 2008-08-14 02:16:30 -0700 | [diff] [blame] | 977 | |
| 978 | nr = 0; |
| 979 | for_each_present_cpu(cpu) { |
| 980 | if (nr >= 8) |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 981 | set_cpu_present(cpu, false); |
Yinghai Lu | a58f03b | 2008-08-14 02:16:30 -0700 | [diff] [blame] | 982 | nr++; |
| 983 | } |
| 984 | |
| 985 | nr = 0; |
| 986 | for_each_possible_cpu(cpu) { |
| 987 | if (nr >= 8) |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 988 | set_cpu_possible(cpu, false); |
Yinghai Lu | a58f03b | 2008-08-14 02:16:30 -0700 | [diff] [blame] | 989 | nr++; |
| 990 | } |
| 991 | |
| 992 | nr_cpu_ids = 8; |
| 993 | } |
| 994 | #endif |
| 995 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 996 | if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) { |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 997 | pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n", |
Michael Tokarev | 55c395b | 2008-12-05 14:42:20 +0300 | [diff] [blame] | 998 | hard_smp_processor_id()); |
| 999 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1000 | physid_set(hard_smp_processor_id(), phys_cpu_present_map); |
| 1001 | } |
| 1002 | |
| 1003 | /* |
| 1004 | * If we couldn't find an SMP configuration at boot time, |
| 1005 | * get out of here now! |
| 1006 | */ |
| 1007 | if (!smp_found_config && !acpi_lapic) { |
Jack Steiner | ac23d4e | 2008-03-28 14:12:16 -0500 | [diff] [blame] | 1008 | preempt_enable(); |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1009 | pr_notice("SMP motherboard not detected\n"); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1010 | disable_smp(); |
| 1011 | if (APIC_init_uniprocessor()) |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1012 | pr_notice("Local APIC not detected. Using dummy APIC emulation.\n"); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1013 | return -1; |
| 1014 | } |
| 1015 | |
| 1016 | /* |
| 1017 | * Should not be necessary because the MP table should list the boot |
| 1018 | * CPU too, but we do it for the sake of robustness anyway. |
| 1019 | */ |
Ingo Molnar | a27a621 | 2009-01-28 12:43:18 +0100 | [diff] [blame] | 1020 | if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) { |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1021 | pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n", |
| 1022 | boot_cpu_physical_apicid); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1023 | physid_set(hard_smp_processor_id(), phys_cpu_present_map); |
| 1024 | } |
Jack Steiner | ac23d4e | 2008-03-28 14:12:16 -0500 | [diff] [blame] | 1025 | preempt_enable(); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1026 | |
| 1027 | /* |
| 1028 | * If we couldn't find a local APIC, then get out of here now! |
| 1029 | */ |
| 1030 | if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) && |
| 1031 | !cpu_has_apic) { |
Cyrill Gorcunov | 103428e | 2009-06-07 16:48:40 +0400 | [diff] [blame] | 1032 | if (!disable_apic) { |
| 1033 | pr_err("BIOS bug, local APIC #%d not detected!...\n", |
| 1034 | boot_cpu_physical_apicid); |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1035 | pr_err("... forcing use of dummy APIC emulation (tell your hw vendor)\n"); |
Cyrill Gorcunov | 103428e | 2009-06-07 16:48:40 +0400 | [diff] [blame] | 1036 | } |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1037 | smpboot_clear_io_apic(); |
Henrik Kretzschmar | 7167d08 | 2011-02-22 15:38:05 +0100 | [diff] [blame] | 1038 | disable_ioapic_support(); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1039 | return -1; |
| 1040 | } |
| 1041 | |
| 1042 | verify_local_APIC(); |
| 1043 | |
| 1044 | /* |
| 1045 | * If SMP should be disabled, then really disable it! |
| 1046 | */ |
| 1047 | if (!max_cpus) { |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1048 | pr_info("SMP mode deactivated\n"); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1049 | smpboot_clear_io_apic(); |
Maciej W. Rozycki | d54db1a | 2008-06-06 03:28:02 +0100 | [diff] [blame] | 1050 | |
Jesse Barnes | e90955c | 2008-04-21 14:14:44 -0700 | [diff] [blame] | 1051 | connect_bsp_APIC(); |
Jesse Barnes | e90955c | 2008-04-21 14:14:44 -0700 | [diff] [blame] | 1052 | setup_local_APIC(); |
Jan Beulich | 2fb270f | 2011-02-09 08:21:02 +0000 | [diff] [blame] | 1053 | bsp_end_local_APIC_setup(); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1054 | return -1; |
| 1055 | } |
| 1056 | |
| 1057 | return 0; |
| 1058 | } |
| 1059 | |
| 1060 | static void __init smp_cpu_index_default(void) |
| 1061 | { |
| 1062 | int i; |
| 1063 | struct cpuinfo_x86 *c; |
| 1064 | |
Akinobu Mita | 7c04e64 | 2008-04-19 23:55:17 +0900 | [diff] [blame] | 1065 | for_each_possible_cpu(i) { |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1066 | c = &cpu_data(i); |
| 1067 | /* mark all to hotplug */ |
Mike Travis | 9628937 | 2008-12-31 18:08:46 -0800 | [diff] [blame] | 1068 | c->cpu_index = nr_cpu_ids; |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1069 | } |
| 1070 | } |
| 1071 | |
| 1072 | /* |
| 1073 | * Prepare for SMP bootup. The MP table or ACPI has been read |
| 1074 | * earlier. Just do some sanity checking here and enable APIC mode. |
| 1075 | */ |
| 1076 | void __init native_smp_prepare_cpus(unsigned int max_cpus) |
| 1077 | { |
Rusty Russell | 7ad728f | 2009-03-13 14:49:50 +1030 | [diff] [blame] | 1078 | unsigned int i; |
| 1079 | |
Ingo Molnar | deef325 | 2008-05-12 15:44:38 +0200 | [diff] [blame] | 1080 | preempt_disable(); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1081 | smp_cpu_index_default(); |
Yinghai Lu | 792363d | 2011-01-21 15:29:54 -0800 | [diff] [blame] | 1082 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1083 | /* |
| 1084 | * Setup boot CPU information |
| 1085 | */ |
Fenghua Yu | 30106c1 | 2012-11-13 11:32:41 -0800 | [diff] [blame] | 1086 | smp_store_boot_cpu_info(); /* Final full version of the data */ |
Yinghai Lu | 792363d | 2011-01-21 15:29:54 -0800 | [diff] [blame] | 1087 | cpumask_copy(cpu_callin_mask, cpumask_of(0)); |
| 1088 | mb(); |
Tejun Heo | bd22a2f | 2011-01-23 14:37:27 +0100 | [diff] [blame] | 1089 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1090 | current_thread_info()->cpu = 0; /* needed? */ |
Rusty Russell | 7ad728f | 2009-03-13 14:49:50 +1030 | [diff] [blame] | 1091 | for_each_possible_cpu(i) { |
Li Zefan | 79f5599 | 2009-06-15 14:58:26 +0800 | [diff] [blame] | 1092 | zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL); |
| 1093 | zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL); |
Yinghai Lu | b3d7336 | 2011-01-21 15:29:44 -0800 | [diff] [blame] | 1094 | zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL); |
Rusty Russell | 7ad728f | 2009-03-13 14:49:50 +1030 | [diff] [blame] | 1095 | } |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1096 | set_cpu_sibling_map(0); |
| 1097 | |
Suresh Siddha | 6e1cb38 | 2008-07-10 11:16:58 -0700 | [diff] [blame] | 1098 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1099 | if (smp_sanity_check(max_cpus) < 0) { |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1100 | pr_info("SMP disabled\n"); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1101 | disable_smp(); |
Ingo Molnar | deef325 | 2008-05-12 15:44:38 +0200 | [diff] [blame] | 1102 | goto out; |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1103 | } |
| 1104 | |
Suresh Siddha | fa47f7e | 2010-08-27 11:09:50 -0700 | [diff] [blame] | 1105 | default_setup_apic_routing(); |
| 1106 | |
Jack Steiner | ac23d4e | 2008-03-28 14:12:16 -0500 | [diff] [blame] | 1107 | preempt_disable(); |
Yinghai Lu | 4c9961d | 2008-07-11 18:44:16 -0700 | [diff] [blame] | 1108 | if (read_apic_id() != boot_cpu_physical_apicid) { |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1109 | panic("Boot APIC ID in local APIC unexpected (%d vs %d)", |
Yinghai Lu | 4c9961d | 2008-07-11 18:44:16 -0700 | [diff] [blame] | 1110 | read_apic_id(), boot_cpu_physical_apicid); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1111 | /* Or can we switch back to PIC here? */ |
| 1112 | } |
Jack Steiner | ac23d4e | 2008-03-28 14:12:16 -0500 | [diff] [blame] | 1113 | preempt_enable(); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1114 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1115 | connect_bsp_APIC(); |
Glauber Costa | b584176 | 2008-05-28 13:38:28 -0300 | [diff] [blame] | 1116 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1117 | /* |
| 1118 | * Switch from PIC to APIC mode. |
| 1119 | */ |
| 1120 | setup_local_APIC(); |
| 1121 | |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 1122 | if (x2apic_mode) |
| 1123 | cpu0_logical_apicid = apic_read(APIC_LDR); |
| 1124 | else |
| 1125 | cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR)); |
| 1126 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1127 | /* |
| 1128 | * Enable IO APIC before setting up error vector |
| 1129 | */ |
| 1130 | if (!skip_ioapic_setup && nr_ioapics) |
| 1131 | enable_IO_APIC(); |
Yinghai Lu | 88d0f55 | 2009-02-14 23:57:28 -0800 | [diff] [blame] | 1132 | |
Jan Beulich | 2fb270f | 2011-02-09 08:21:02 +0000 | [diff] [blame] | 1133 | bsp_end_local_APIC_setup(); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1134 | |
Ingo Molnar | d83093b | 2009-01-28 12:43:18 +0100 | [diff] [blame] | 1135 | if (apic->setup_portio_remap) |
| 1136 | apic->setup_portio_remap(); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1137 | |
| 1138 | smpboot_setup_io_apic(); |
| 1139 | /* |
| 1140 | * Set up local APIC timer on boot CPU. |
| 1141 | */ |
| 1142 | |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1143 | pr_info("CPU%d: ", 0); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1144 | print_cpu_info(&cpu_data(0)); |
Thomas Gleixner | 736deca | 2009-08-19 12:35:53 +0200 | [diff] [blame] | 1145 | x86_init.timers.setup_percpu_clockev(); |
Marcin Slusarz | c4bd1fd | 2008-08-21 20:49:05 +0200 | [diff] [blame] | 1146 | |
| 1147 | if (is_uv_system()) |
| 1148 | uv_system_init(); |
Suresh Siddha | d0af9ee | 2009-08-19 18:05:36 -0700 | [diff] [blame] | 1149 | |
| 1150 | set_mtrr_aps_delayed_init(); |
Ingo Molnar | deef325 | 2008-05-12 15:44:38 +0200 | [diff] [blame] | 1151 | out: |
| 1152 | preempt_enable(); |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1153 | } |
Suresh Siddha | d0af9ee | 2009-08-19 18:05:36 -0700 | [diff] [blame] | 1154 | |
| 1155 | void arch_enable_nonboot_cpus_begin(void) |
| 1156 | { |
| 1157 | set_mtrr_aps_delayed_init(); |
| 1158 | } |
| 1159 | |
| 1160 | void arch_enable_nonboot_cpus_end(void) |
| 1161 | { |
| 1162 | mtrr_aps_init(); |
| 1163 | } |
| 1164 | |
Glauber de Oliveira Costa | 8aef135 | 2008-03-19 14:26:11 -0300 | [diff] [blame] | 1165 | /* |
Glauber de Oliveira Costa | a8db845 | 2008-03-19 14:26:01 -0300 | [diff] [blame] | 1166 | * Early setup to make printk work. |
| 1167 | */ |
| 1168 | void __init native_smp_prepare_boot_cpu(void) |
| 1169 | { |
| 1170 | int me = smp_processor_id(); |
Brian Gerst | 552be87 | 2009-01-30 17:47:53 +0900 | [diff] [blame] | 1171 | switch_to_new_gdt(me); |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 1172 | /* already set me in cpu_online_mask in boot_cpu_init() */ |
| 1173 | cpumask_set_cpu(me, cpu_callout_mask); |
Glauber de Oliveira Costa | a8db845 | 2008-03-19 14:26:01 -0300 | [diff] [blame] | 1174 | per_cpu(cpu_state, me) = CPU_ONLINE; |
| 1175 | } |
| 1176 | |
Glauber de Oliveira Costa | 83f7eb9 | 2008-03-19 14:26:02 -0300 | [diff] [blame] | 1177 | void __init native_smp_cpus_done(unsigned int max_cpus) |
| 1178 | { |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1179 | pr_debug("Boot done\n"); |
Glauber de Oliveira Costa | 83f7eb9 | 2008-03-19 14:26:02 -0300 | [diff] [blame] | 1180 | |
Don Zickus | 99e8b9c | 2011-10-13 15:14:26 -0400 | [diff] [blame] | 1181 | nmi_selftest(); |
Glauber de Oliveira Costa | 83f7eb9 | 2008-03-19 14:26:02 -0300 | [diff] [blame] | 1182 | impress_friends(); |
Glauber de Oliveira Costa | 83f7eb9 | 2008-03-19 14:26:02 -0300 | [diff] [blame] | 1183 | #ifdef CONFIG_X86_IO_APIC |
| 1184 | setup_ioapic_dest(); |
| 1185 | #endif |
Suresh Siddha | d0af9ee | 2009-08-19 18:05:36 -0700 | [diff] [blame] | 1186 | mtrr_aps_init(); |
Glauber de Oliveira Costa | 83f7eb9 | 2008-03-19 14:26:02 -0300 | [diff] [blame] | 1187 | } |
| 1188 | |
Mike Travis | 3b11ce7 | 2008-12-17 15:21:39 -0800 | [diff] [blame] | 1189 | static int __initdata setup_possible_cpus = -1; |
| 1190 | static int __init _setup_possible_cpus(char *str) |
| 1191 | { |
| 1192 | get_option(&str, &setup_possible_cpus); |
| 1193 | return 0; |
| 1194 | } |
| 1195 | early_param("possible_cpus", _setup_possible_cpus); |
| 1196 | |
| 1197 | |
Glauber Costa | 68a1c3f | 2008-03-03 14:12:42 -0300 | [diff] [blame] | 1198 | /* |
Rusty Russell | 4f06289 | 2009-03-13 14:49:54 +1030 | [diff] [blame] | 1199 | * cpu_possible_mask should be static, it cannot change as cpu's |
Glauber Costa | 68a1c3f | 2008-03-03 14:12:42 -0300 | [diff] [blame] | 1200 | * are onlined, or offlined. The reason is per-cpu data-structures |
| 1201 | * are allocated by some modules at init time, and dont expect to |
| 1202 | * do this dynamically on cpu arrival/departure. |
Rusty Russell | 4f06289 | 2009-03-13 14:49:54 +1030 | [diff] [blame] | 1203 | * cpu_present_mask on the other hand can change dynamically. |
Glauber Costa | 68a1c3f | 2008-03-03 14:12:42 -0300 | [diff] [blame] | 1204 | * In case when cpu_hotplug is not compiled, then we resort to current |
| 1205 | * behaviour, which is cpu_possible == cpu_present. |
| 1206 | * - Ashok Raj |
| 1207 | * |
| 1208 | * Three ways to find out the number of additional hotplug CPUs: |
| 1209 | * - If the BIOS specified disabled CPUs in ACPI/mptables use that. |
Mike Travis | 3b11ce7 | 2008-12-17 15:21:39 -0800 | [diff] [blame] | 1210 | * - The user can overwrite it with possible_cpus=NUM |
Glauber Costa | 68a1c3f | 2008-03-03 14:12:42 -0300 | [diff] [blame] | 1211 | * - Otherwise don't reserve additional CPUs. |
| 1212 | * We do this because additional CPUs waste a lot of memory. |
| 1213 | * -AK |
| 1214 | */ |
| 1215 | __init void prefill_possible_map(void) |
| 1216 | { |
Thomas Gleixner | cb48bb5 | 2008-10-05 17:51:52 +0200 | [diff] [blame] | 1217 | int i, possible; |
Glauber Costa | 68a1c3f | 2008-03-03 14:12:42 -0300 | [diff] [blame] | 1218 | |
Yinghai Lu | 329513a | 2008-07-02 18:54:40 -0700 | [diff] [blame] | 1219 | /* no processor from mptable or madt */ |
| 1220 | if (!num_processors) |
| 1221 | num_processors = 1; |
| 1222 | |
Jan Beulich | 5f2eb55 | 2010-05-24 12:13:17 -0700 | [diff] [blame] | 1223 | i = setup_max_cpus ?: 1; |
| 1224 | if (setup_possible_cpus == -1) { |
| 1225 | possible = num_processors; |
| 1226 | #ifdef CONFIG_HOTPLUG_CPU |
| 1227 | if (setup_max_cpus) |
| 1228 | possible += disabled_cpus; |
| 1229 | #else |
| 1230 | if (possible > i) |
| 1231 | possible = i; |
| 1232 | #endif |
| 1233 | } else |
Mike Travis | 3b11ce7 | 2008-12-17 15:21:39 -0800 | [diff] [blame] | 1234 | possible = setup_possible_cpus; |
| 1235 | |
Mike Travis | 730cf27 | 2008-12-31 18:08:45 -0800 | [diff] [blame] | 1236 | total_cpus = max_t(int, possible, num_processors + disabled_cpus); |
| 1237 | |
Yinghai Lu | 2b633e3 | 2010-02-10 01:20:37 -0800 | [diff] [blame] | 1238 | /* nr_cpu_ids could be reduced via nr_cpus= */ |
| 1239 | if (possible > nr_cpu_ids) { |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1240 | pr_warn("%d Processors exceeds NR_CPUS limit of %d\n", |
Yinghai Lu | 2b633e3 | 2010-02-10 01:20:37 -0800 | [diff] [blame] | 1241 | possible, nr_cpu_ids); |
| 1242 | possible = nr_cpu_ids; |
Mike Travis | 3b11ce7 | 2008-12-17 15:21:39 -0800 | [diff] [blame] | 1243 | } |
Glauber Costa | 68a1c3f | 2008-03-03 14:12:42 -0300 | [diff] [blame] | 1244 | |
Jan Beulich | 5f2eb55 | 2010-05-24 12:13:17 -0700 | [diff] [blame] | 1245 | #ifdef CONFIG_HOTPLUG_CPU |
| 1246 | if (!setup_max_cpus) |
| 1247 | #endif |
| 1248 | if (possible > i) { |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1249 | pr_warn("%d Processors exceeds max_cpus limit of %u\n", |
Jan Beulich | 5f2eb55 | 2010-05-24 12:13:17 -0700 | [diff] [blame] | 1250 | possible, setup_max_cpus); |
| 1251 | possible = i; |
| 1252 | } |
| 1253 | |
Joe Perches | c767a54 | 2012-05-21 19:50:07 -0700 | [diff] [blame] | 1254 | pr_info("Allowing %d CPUs, %d hotplug CPUs\n", |
Glauber Costa | 68a1c3f | 2008-03-03 14:12:42 -0300 | [diff] [blame] | 1255 | possible, max_t(int, possible - num_processors, 0)); |
| 1256 | |
| 1257 | for (i = 0; i < possible; i++) |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 1258 | set_cpu_possible(i, true); |
Jan Beulich | 5f2eb55 | 2010-05-24 12:13:17 -0700 | [diff] [blame] | 1259 | for (; i < NR_CPUS; i++) |
| 1260 | set_cpu_possible(i, false); |
Mike Travis | 3461b0a | 2008-05-12 21:21:13 +0200 | [diff] [blame] | 1261 | |
| 1262 | nr_cpu_ids = possible; |
Glauber Costa | 68a1c3f | 2008-03-03 14:12:42 -0300 | [diff] [blame] | 1263 | } |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1264 | |
Chuck Ebbert | 14adf85 | 2008-09-29 18:29:42 -0400 | [diff] [blame] | 1265 | #ifdef CONFIG_HOTPLUG_CPU |
| 1266 | |
| 1267 | static void remove_siblinginfo(int cpu) |
| 1268 | { |
| 1269 | int sibling; |
| 1270 | struct cpuinfo_x86 *c = &cpu_data(cpu); |
| 1271 | |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 1272 | for_each_cpu(sibling, cpu_core_mask(cpu)) { |
| 1273 | cpumask_clear_cpu(cpu, cpu_core_mask(sibling)); |
Chuck Ebbert | 14adf85 | 2008-09-29 18:29:42 -0400 | [diff] [blame] | 1274 | /*/ |
| 1275 | * last thread sibling in this cpu core going down |
| 1276 | */ |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 1277 | if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) |
Chuck Ebbert | 14adf85 | 2008-09-29 18:29:42 -0400 | [diff] [blame] | 1278 | cpu_data(sibling).booted_cores--; |
| 1279 | } |
| 1280 | |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 1281 | for_each_cpu(sibling, cpu_sibling_mask(cpu)) |
| 1282 | cpumask_clear_cpu(cpu, cpu_sibling_mask(sibling)); |
| 1283 | cpumask_clear(cpu_sibling_mask(cpu)); |
| 1284 | cpumask_clear(cpu_core_mask(cpu)); |
Chuck Ebbert | 14adf85 | 2008-09-29 18:29:42 -0400 | [diff] [blame] | 1285 | c->phys_proc_id = 0; |
| 1286 | c->cpu_core_id = 0; |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 1287 | cpumask_clear_cpu(cpu, cpu_sibling_setup_mask); |
Chuck Ebbert | 14adf85 | 2008-09-29 18:29:42 -0400 | [diff] [blame] | 1288 | } |
| 1289 | |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1290 | static void __ref remove_cpu_from_maps(int cpu) |
| 1291 | { |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 1292 | set_cpu_online(cpu, false); |
| 1293 | cpumask_clear_cpu(cpu, cpu_callout_mask); |
| 1294 | cpumask_clear_cpu(cpu, cpu_callin_mask); |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1295 | /* was set by cpu_init() */ |
Mike Travis | c2d1cec | 2009-01-04 05:18:03 -0800 | [diff] [blame] | 1296 | cpumask_clear_cpu(cpu, cpu_initialized_mask); |
Mike Travis | 23ca4bb | 2008-05-12 21:21:12 +0200 | [diff] [blame] | 1297 | numa_remove_cpu(cpu); |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1298 | } |
| 1299 | |
Alex Nixon | 8227dce | 2008-08-22 11:52:14 +0100 | [diff] [blame] | 1300 | void cpu_disable_common(void) |
| 1301 | { |
| 1302 | int cpu = smp_processor_id(); |
Alex Nixon | 8227dce | 2008-08-22 11:52:14 +0100 | [diff] [blame] | 1303 | |
Alex Nixon | 8227dce | 2008-08-22 11:52:14 +0100 | [diff] [blame] | 1304 | remove_siblinginfo(cpu); |
| 1305 | |
| 1306 | /* It's now safe to remove this processor from the online map */ |
| 1307 | lock_vector_lock(); |
| 1308 | remove_cpu_from_maps(cpu); |
| 1309 | unlock_vector_lock(); |
Mike Travis | d7b381b | 2008-12-16 17:33:58 -0800 | [diff] [blame] | 1310 | fixup_irqs(); |
Alex Nixon | 8227dce | 2008-08-22 11:52:14 +0100 | [diff] [blame] | 1311 | } |
| 1312 | |
Alex Nixon | 93be71b | 2008-08-22 11:52:11 +0100 | [diff] [blame] | 1313 | int native_cpu_disable(void) |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1314 | { |
Prarit Bhargava | da6139e | 2014-01-13 06:51:01 -0500 | [diff] [blame] | 1315 | int ret; |
| 1316 | |
| 1317 | ret = check_irq_vectors_for_cpu_disable(); |
| 1318 | if (ret) |
| 1319 | return ret; |
| 1320 | |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1321 | clear_local_APIC(); |
| 1322 | |
Alex Nixon | 8227dce | 2008-08-22 11:52:14 +0100 | [diff] [blame] | 1323 | cpu_disable_common(); |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1324 | return 0; |
| 1325 | } |
| 1326 | |
Alex Nixon | 93be71b | 2008-08-22 11:52:11 +0100 | [diff] [blame] | 1327 | void native_cpu_die(unsigned int cpu) |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1328 | { |
| 1329 | /* We don't do anything here: idle task is faking death itself. */ |
| 1330 | unsigned int i; |
| 1331 | |
| 1332 | for (i = 0; i < 10; i++) { |
| 1333 | /* They ack this in play_dead by setting CPU_DEAD */ |
| 1334 | if (per_cpu(cpu_state, cpu) == CPU_DEAD) { |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 1335 | if (system_state == SYSTEM_RUNNING) |
| 1336 | pr_info("CPU %u is now offline\n", cpu); |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1337 | return; |
| 1338 | } |
| 1339 | msleep(100); |
| 1340 | } |
Mike Travis | 2eaad1f | 2009-12-10 17:19:36 -0800 | [diff] [blame] | 1341 | pr_err("CPU %u didn't die...\n", cpu); |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1342 | } |
Alex Nixon | a21f5d88 | 2008-08-22 11:52:13 +0100 | [diff] [blame] | 1343 | |
| 1344 | void play_dead_common(void) |
| 1345 | { |
| 1346 | idle_task_exit(); |
| 1347 | reset_lazy_tlbstate(); |
Len Brown | 02c68a0 | 2011-04-01 16:59:53 -0400 | [diff] [blame] | 1348 | amd_e400_remove_cpu(raw_smp_processor_id()); |
Alex Nixon | a21f5d88 | 2008-08-22 11:52:13 +0100 | [diff] [blame] | 1349 | |
| 1350 | mb(); |
| 1351 | /* Ack it */ |
Tejun Heo | 0a3aee0 | 2010-12-18 16:28:55 +0100 | [diff] [blame] | 1352 | __this_cpu_write(cpu_state, CPU_DEAD); |
Alex Nixon | a21f5d88 | 2008-08-22 11:52:13 +0100 | [diff] [blame] | 1353 | |
| 1354 | /* |
| 1355 | * With physical CPU hotplug, we should halt the cpu |
| 1356 | */ |
| 1357 | local_irq_disable(); |
| 1358 | } |
| 1359 | |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 1360 | static bool wakeup_cpu0(void) |
| 1361 | { |
| 1362 | if (smp_processor_id() == 0 && enable_start_cpu0) |
| 1363 | return true; |
| 1364 | |
| 1365 | return false; |
| 1366 | } |
| 1367 | |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1368 | /* |
| 1369 | * We need to flush the caches before going to sleep, lest we have |
| 1370 | * dirty data in our caches when we come back up. |
| 1371 | */ |
| 1372 | static inline void mwait_play_dead(void) |
| 1373 | { |
| 1374 | unsigned int eax, ebx, ecx, edx; |
| 1375 | unsigned int highest_cstate = 0; |
| 1376 | unsigned int highest_subcstate = 0; |
H. Peter Anvin | ce5f682 | 2010-09-20 13:04:45 -0700 | [diff] [blame] | 1377 | void *mwait_ptr; |
Borislav Petkov | 576cfb4 | 2013-03-04 21:16:16 +0100 | [diff] [blame] | 1378 | int i; |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1379 | |
Len Brown | 69fb367 | 2013-02-10 01:38:39 -0500 | [diff] [blame] | 1380 | if (!this_cpu_has(X86_FEATURE_MWAIT)) |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1381 | return; |
Christoph Lameter | 349c004 | 2011-03-12 12:50:10 +0100 | [diff] [blame] | 1382 | if (!this_cpu_has(X86_FEATURE_CLFLSH)) |
H. Peter Anvin | ce5f682 | 2010-09-20 13:04:45 -0700 | [diff] [blame] | 1383 | return; |
Tejun Heo | 7b543a5 | 2010-12-18 16:30:05 +0100 | [diff] [blame] | 1384 | if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF) |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1385 | return; |
| 1386 | |
| 1387 | eax = CPUID_MWAIT_LEAF; |
| 1388 | ecx = 0; |
| 1389 | native_cpuid(&eax, &ebx, &ecx, &edx); |
| 1390 | |
| 1391 | /* |
| 1392 | * eax will be 0 if EDX enumeration is not valid. |
| 1393 | * Initialized below to cstate, sub_cstate value when EDX is valid. |
| 1394 | */ |
| 1395 | if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) { |
| 1396 | eax = 0; |
| 1397 | } else { |
| 1398 | edx >>= MWAIT_SUBSTATE_SIZE; |
| 1399 | for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) { |
| 1400 | if (edx & MWAIT_SUBSTATE_MASK) { |
| 1401 | highest_cstate = i; |
| 1402 | highest_subcstate = edx & MWAIT_SUBSTATE_MASK; |
| 1403 | } |
| 1404 | } |
| 1405 | eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) | |
| 1406 | (highest_subcstate - 1); |
| 1407 | } |
| 1408 | |
H. Peter Anvin | ce5f682 | 2010-09-20 13:04:45 -0700 | [diff] [blame] | 1409 | /* |
| 1410 | * This should be a memory location in a cache line which is |
| 1411 | * unlikely to be touched by other processors. The actual |
| 1412 | * content is immaterial as it is not actually modified in any way. |
| 1413 | */ |
| 1414 | mwait_ptr = ¤t_thread_info()->flags; |
| 1415 | |
H. Peter Anvin | a68e5c9 | 2010-09-17 17:06:46 -0700 | [diff] [blame] | 1416 | wbinvd(); |
| 1417 | |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1418 | while (1) { |
H. Peter Anvin | ce5f682 | 2010-09-20 13:04:45 -0700 | [diff] [blame] | 1419 | /* |
| 1420 | * The CLFLUSH is a workaround for erratum AAI65 for |
| 1421 | * the Xeon 7400 series. It's not clear it is actually |
| 1422 | * needed, but it should be harmless in either case. |
| 1423 | * The WBINVD is insufficient due to the spurious-wakeup |
| 1424 | * case where we return around the loop. |
| 1425 | */ |
H. Peter Anvin | 7d590cc | 2013-12-19 12:30:03 -0800 | [diff] [blame] | 1426 | mb(); |
H. Peter Anvin | ce5f682 | 2010-09-20 13:04:45 -0700 | [diff] [blame] | 1427 | clflush(mwait_ptr); |
H. Peter Anvin | 7d590cc | 2013-12-19 12:30:03 -0800 | [diff] [blame] | 1428 | mb(); |
H. Peter Anvin | ce5f682 | 2010-09-20 13:04:45 -0700 | [diff] [blame] | 1429 | __monitor(mwait_ptr, 0, 0); |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1430 | mb(); |
| 1431 | __mwait(eax, 0); |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 1432 | /* |
| 1433 | * If NMI wants to wake up CPU0, start CPU0. |
| 1434 | */ |
| 1435 | if (wakeup_cpu0()) |
| 1436 | start_cpu0(); |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1437 | } |
| 1438 | } |
| 1439 | |
| 1440 | static inline void hlt_play_dead(void) |
| 1441 | { |
Tejun Heo | 7b543a5 | 2010-12-18 16:30:05 +0100 | [diff] [blame] | 1442 | if (__this_cpu_read(cpu_info.x86) >= 4) |
H. Peter Anvin | a68e5c9 | 2010-09-17 17:06:46 -0700 | [diff] [blame] | 1443 | wbinvd(); |
| 1444 | |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1445 | while (1) { |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1446 | native_halt(); |
Fenghua Yu | e1c467e | 2012-11-14 04:36:53 -0800 | [diff] [blame] | 1447 | /* |
| 1448 | * If NMI wants to wake up CPU0, start CPU0. |
| 1449 | */ |
| 1450 | if (wakeup_cpu0()) |
| 1451 | start_cpu0(); |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1452 | } |
| 1453 | } |
| 1454 | |
Alex Nixon | a21f5d88 | 2008-08-22 11:52:13 +0100 | [diff] [blame] | 1455 | void native_play_dead(void) |
| 1456 | { |
| 1457 | play_dead_common(); |
Joseph Cihula | 86886e5 | 2009-06-30 19:31:07 -0700 | [diff] [blame] | 1458 | tboot_shutdown(TB_SHUTDOWN_WFS); |
H. Peter Anvin | ea53069 | 2010-09-17 15:39:11 -0700 | [diff] [blame] | 1459 | |
| 1460 | mwait_play_dead(); /* Only returns on failure */ |
Boris Ostrovsky | 1a022e3 | 2012-03-13 19:55:09 +0100 | [diff] [blame] | 1461 | if (cpuidle_play_dead()) |
| 1462 | hlt_play_dead(); |
Alex Nixon | a21f5d88 | 2008-08-22 11:52:13 +0100 | [diff] [blame] | 1463 | } |
| 1464 | |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1465 | #else /* ... !CONFIG_HOTPLUG_CPU */ |
Alex Nixon | 93be71b | 2008-08-22 11:52:11 +0100 | [diff] [blame] | 1466 | int native_cpu_disable(void) |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1467 | { |
| 1468 | return -ENOSYS; |
| 1469 | } |
| 1470 | |
Alex Nixon | 93be71b | 2008-08-22 11:52:11 +0100 | [diff] [blame] | 1471 | void native_cpu_die(unsigned int cpu) |
Glauber Costa | 69c18c1 | 2008-03-03 14:13:07 -0300 | [diff] [blame] | 1472 | { |
| 1473 | /* We said "no" in __cpu_disable */ |
| 1474 | BUG(); |
| 1475 | } |
Alex Nixon | a21f5d88 | 2008-08-22 11:52:13 +0100 | [diff] [blame] | 1476 | |
| 1477 | void native_play_dead(void) |
| 1478 | { |
| 1479 | BUG(); |
| 1480 | } |
| 1481 | |
Glauber Costa | 68a1c3f | 2008-03-03 14:12:42 -0300 | [diff] [blame] | 1482 | #endif |