blob: dcb52850a28fcbe00a4a25ddf47d6f4ccedf3c9c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Local APIC handling, local APIC timers
3 *
Ingo Molnar8f47e162009-01-31 02:03:42 +01004 * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * Fixes
7 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
8 * thanks to Eric Gilmore
9 * and Rolf G. Tews
10 * for testing these extensively.
11 * Maciej W. Rozycki : Various updates and fixes.
12 * Mikael Pettersson : Power Management for UP-APIC.
13 * Pavel Machek and
14 * Mikael Pettersson : PM converted to driver model.
15 */
16
Ingo Molnarcdd6c482009-09-21 12:02:48 +020017#include <linux/perf_event.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/kernel_stat.h>
Ingo Molnard1de36f2009-01-31 01:59:14 +010019#include <linux/mc146818rtc.h>
Thomas Gleixner70a20022008-01-30 13:30:18 +010020#include <linux/acpi_pmtmr.h>
Ingo Molnard1de36f2009-01-31 01:59:14 +010021#include <linux/clockchips.h>
22#include <linux/interrupt.h>
23#include <linux/bootmem.h>
Frederic Weisbeckerbcbc4f22008-12-09 23:54:20 +010024#include <linux/ftrace.h>
Ingo Molnard1de36f2009-01-31 01:59:14 +010025#include <linux/ioport.h>
26#include <linux/module.h>
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +010027#include <linux/syscore_ops.h>
Ingo Molnard1de36f2009-01-31 01:59:14 +010028#include <linux/delay.h>
Jaswinder Singh Rajpute423e332009-01-04 16:16:25 +053029#include <linux/timex.h>
Ralf Baechle334955e2011-06-01 19:04:57 +010030#include <linux/i8253.h>
Ingo Molnard1de36f2009-01-31 01:59:14 +010031#include <linux/dmar.h>
32#include <linux/init.h>
33#include <linux/cpu.h>
34#include <linux/dmi.h>
Ingo Molnard1de36f2009-01-31 01:59:14 +010035#include <linux/smp.h>
36#include <linux/mm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
Steven Rostedt (Red Hat)83ab8512013-06-21 10:29:05 -040038#include <asm/trace/irq_vectors.h>
Suresh Siddha8a8f4222012-03-30 11:47:08 -070039#include <asm/irq_remapping.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020040#include <asm/perf_event.h>
Thomas Gleixner736deca2009-08-19 12:35:53 +020041#include <asm/x86_init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070042#include <asm/pgalloc.h>
Arun Sharma600634972011-07-26 16:09:06 -070043#include <linux/atomic.h>
Ingo Molnard1de36f2009-01-31 01:59:14 +010044#include <asm/mpspec.h>
Ingo Molnard1de36f2009-01-31 01:59:14 +010045#include <asm/i8259.h>
Andi Kleen73dea472006-02-03 21:50:50 +010046#include <asm/proto.h>
Andi Kleen2c8c0e62006-09-26 10:52:32 +020047#include <asm/apic.h>
Henrik Kretzschmar7167d082011-02-22 15:38:05 +010048#include <asm/io_apic.h>
Ingo Molnard1de36f2009-01-31 01:59:14 +010049#include <asm/desc.h>
50#include <asm/hpet.h>
51#include <asm/idle.h>
52#include <asm/mtrr.h>
Ralf Baechle16f871b2011-06-01 19:05:06 +010053#include <asm/time.h>
Jaswinder Singh Rajput2bc13792009-01-11 20:34:47 +053054#include <asm/smp.h>
Andi Kleenbe71b852009-02-12 13:49:38 +010055#include <asm/mce.h>
Kerstin Jonsson8c3ba8d2010-05-24 12:13:15 -070056#include <asm/tsc.h>
Sheng Yang2904ed82010-12-21 14:18:48 +080057#include <asm/hypervisor.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070058
Brian Gerstec70de82009-01-27 12:56:47 +090059unsigned int num_processors;
Ingo Molnarfdbecd92009-01-31 03:57:12 +010060
Paul Gortmaker148f9bb2013-06-18 18:23:59 -040061unsigned disabled_cpus;
Ingo Molnarfdbecd92009-01-31 03:57:12 +010062
Brian Gerstec70de82009-01-27 12:56:47 +090063/* Processor that is doing the boot up */
64unsigned int boot_cpu_physical_apicid = -1U;
David Rientjescc08e042013-11-14 15:05:32 -080065EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
Glauber Costa5af55732008-03-25 13:28:56 -030066
Cyrill Gorcunov80e56092008-08-24 02:01:42 -070067/*
Ingo Molnarfdbecd92009-01-31 03:57:12 +010068 * The highest APIC ID seen during enumeration.
Cyrill Gorcunov80e56092008-08-24 02:01:42 -070069 */
Jiang Liua491cc9022014-06-09 16:19:32 +080070static unsigned int max_physical_apicid;
Brian Gerstec70de82009-01-27 12:56:47 +090071
Ingo Molnarfdbecd92009-01-31 03:57:12 +010072/*
73 * Bitmask of physically existing CPUs:
74 */
Brian Gerstec70de82009-01-27 12:56:47 +090075physid_mask_t phys_cpu_present_map;
76
77/*
HATAYAMA Daisuke151e0c72014-01-15 15:44:58 +090078 * Processor to be disabled specified by kernel parameter
79 * disable_cpu_apicid=<int>, mostly used for the kdump 2nd kernel to
80 * avoid undefined behaviour caused by sending INIT from AP to BSP.
81 */
H. Peter Anvin5b4d1db2014-01-15 13:02:08 -080082static unsigned int disabled_cpu_apicid __read_mostly = BAD_APICID;
HATAYAMA Daisuke151e0c72014-01-15 15:44:58 +090083
84/*
Brian Gerstec70de82009-01-27 12:56:47 +090085 * Map cpu index to physical APIC ID
86 */
Vlad Zolotarov0816b0f2012-06-11 12:56:52 +030087DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);
88DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID);
Brian Gerstec70de82009-01-27 12:56:47 +090089EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
90EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
Cyrill Gorcunov80e56092008-08-24 02:01:42 -070091
Yinghai Lub3c51172008-08-24 02:01:46 -070092#ifdef CONFIG_X86_32
Tejun Heo4c321ff2011-01-23 14:37:30 +010093
Tejun Heo4c321ff2011-01-23 14:37:30 +010094/*
95 * On x86_32, the mapping between cpu and logical apicid may vary
96 * depending on apic in use. The following early percpu variable is
97 * used for the mapping. This is where the behaviors of x86_64 and 32
98 * actually diverge. Let's keep it ugly for now.
99 */
Vlad Zolotarov0816b0f2012-06-11 12:56:52 +0300100DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID);
Tejun Heo4c321ff2011-01-23 14:37:30 +0100101
Yinghai Luf28c0ae2008-08-24 02:01:49 -0700102/* Local APIC was disabled by the BIOS and enabled by the kernel */
103static int enabled_via_apicbase;
104
Cyrill Gorcunovc0eaa452009-04-12 20:47:40 +0400105/*
106 * Handle interrupt mode configuration register (IMCR).
107 * This register controls whether the interrupt signals
108 * that reach the BSP come from the master PIC or from the
109 * local APIC. Before entering Symmetric I/O Mode, either
110 * the BIOS or the operating system must switch out of
111 * PIC Mode by changing the IMCR.
112 */
Alexander van Heukelum5cda3952009-04-13 17:39:24 +0200113static inline void imcr_pic_to_apic(void)
Cyrill Gorcunovc0eaa452009-04-12 20:47:40 +0400114{
115 /* select IMCR register */
116 outb(0x70, 0x22);
117 /* NMI and 8259 INTR go through APIC */
118 outb(0x01, 0x23);
119}
120
Alexander van Heukelum5cda3952009-04-13 17:39:24 +0200121static inline void imcr_apic_to_pic(void)
Cyrill Gorcunovc0eaa452009-04-12 20:47:40 +0400122{
123 /* select IMCR register */
124 outb(0x70, 0x22);
125 /* NMI and 8259 INTR go directly to BSP */
126 outb(0x00, 0x23);
127}
Yinghai Lub3c51172008-08-24 02:01:46 -0700128#endif
129
Suresh Siddha279f1462012-10-22 14:37:58 -0700130/*
131 * Knob to control our willingness to enable the local APIC.
132 *
133 * +1=force-enable
134 */
135static int force_enable_local_apic __initdata;
David Rientjesdc9788f2014-02-04 23:55:06 -0800136
Suresh Siddha279f1462012-10-22 14:37:58 -0700137/*
138 * APIC command line parameters
139 */
140static int __init parse_lapic(char *arg)
141{
142 if (config_enabled(CONFIG_X86_32) && !arg)
143 force_enable_local_apic = 1;
Mathias Krause27cf9292013-02-19 20:47:07 +0100144 else if (arg && !strncmp(arg, "notscdeadline", 13))
Suresh Siddha279f1462012-10-22 14:37:58 -0700145 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
146 return 0;
147}
148early_param("lapic", parse_lapic);
149
Yinghai Lub3c51172008-08-24 02:01:46 -0700150#ifdef CONFIG_X86_64
Chris Wrightbc1d99c2007-10-12 23:04:23 +0200151static int apic_calibrate_pmtmr __initdata;
Yinghai Lub3c51172008-08-24 02:01:46 -0700152static __init int setup_apicpmtimer(char *s)
153{
154 apic_calibrate_pmtmr = 1;
155 notsc_setup(NULL);
156 return 0;
157}
158__setup("apicpmtimer", setup_apicpmtimer);
159#endif
160
Yinghai Lub3c51172008-08-24 02:01:46 -0700161unsigned long mp_lapic_addr;
162int disable_apic;
163/* Disable local APIC timer from the kernel commandline or via dmi quirk */
Henrik Kretzschmar25874a22011-03-11 08:02:36 +0100164static int disable_apic_timer __initdata;
Hiroshi Shimamotoe83a5fd2008-01-30 13:32:35 +0100165/* Local APIC timer works in C2 */
Linus Torvalds2e7c2832007-03-23 11:32:31 -0700166int local_apic_timer_c2_ok;
167EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
168
Jan Beulich2414e022014-11-03 08:39:43 +0000169int first_system_vector = FIRST_SYSTEM_VECTOR;
Yinghai Luefa25592008-08-19 20:50:36 -0700170
Hiroshi Shimamotoe83a5fd2008-01-30 13:32:35 +0100171/*
172 * Debug level, exported for io_apic.c
173 */
Maciej W. Rozyckibaa13182008-07-14 18:44:51 +0100174unsigned int apic_verbosity;
Hiroshi Shimamotoe83a5fd2008-01-30 13:32:35 +0100175
Cyrill Gorcunov89c38c22008-08-24 02:01:43 -0700176int pic_mode;
177
Alexey Starikovskiybab4b272008-05-19 19:47:03 +0400178/* Have we found an MP table */
179int smp_found_config;
180
Aaron Durbin39928722006-12-07 02:14:01 +0100181static struct resource lapic_resource = {
182 .name = "Local APIC",
183 .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
184};
185
Jacob Pan1ade93e2011-11-10 13:42:40 +0000186unsigned int lapic_timer_frequency = 0;
Thomas Gleixnerd03030e2007-10-12 23:04:06 +0200187
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100188static void apic_pm_activate(void);
Thomas Gleixnerba7eda42007-10-12 23:04:07 +0200189
Andi Kleend3432892008-01-30 13:33:17 +0100190static unsigned long apic_phys;
191
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100192/*
193 * Get the LAPIC version
194 */
195static inline int lapic_get_version(void)
196{
197 return GET_APIC_VERSION(apic_read(APIC_LVR));
198}
199
200/*
Cyrill Gorcunov9c803862008-08-16 23:21:54 +0400201 * Check, if the APIC is integrated or a separate chip
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100202 */
203static inline int lapic_is_integrated(void)
204{
Cyrill Gorcunov9c803862008-08-16 23:21:54 +0400205#ifdef CONFIG_X86_64
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100206 return 1;
Cyrill Gorcunov9c803862008-08-16 23:21:54 +0400207#else
208 return APIC_INTEGRATED(lapic_get_version());
209#endif
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100210}
211
212/*
213 * Check, whether this is a modern or a first generation APIC
214 */
215static int modern_apic(void)
216{
217 /* AMD systems use old APIC versions, so check the CPU */
218 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
219 boot_cpu_data.x86 >= 0xf)
220 return 1;
221 return lapic_get_version() >= 0x14;
222}
223
Cyrill Gorcunov08306ce2009-04-12 20:47:41 +0400224/*
Cyrill Gorcunova933c612009-10-14 00:07:04 +0400225 * right after this call apic become NOOP driven
226 * so apic->write/read doesn't do anything
Cyrill Gorcunov08306ce2009-04-12 20:47:41 +0400227 */
Henrik Kretzschmar25874a22011-03-11 08:02:36 +0100228static void __init apic_disable(void)
Cyrill Gorcunov08306ce2009-04-12 20:47:41 +0400229{
Cyrill Gorcunovf88f2b42009-10-15 19:04:16 +0400230 pr_info("APIC: switched to apic NOOP\n");
Cyrill Gorcunova933c612009-10-14 00:07:04 +0400231 apic = &apic_noop;
Cyrill Gorcunov08306ce2009-04-12 20:47:41 +0400232}
233
Yinghai Luc1eeb2d2009-02-16 23:02:14 -0800234void native_apic_wait_icr_idle(void)
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100235{
236 while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
237 cpu_relax();
238}
239
Yinghai Luc1eeb2d2009-02-16 23:02:14 -0800240u32 native_safe_apic_wait_icr_idle(void)
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100241{
242 u32 send_status;
243 int timeout;
244
245 timeout = 0;
246 do {
247 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
248 if (!send_status)
249 break;
Fernando Luis Vazquez Caob49d7d82011-12-15 11:32:24 +0900250 inc_irq_stat(icr_read_retry_count);
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100251 udelay(100);
252 } while (timeout++ < 1000);
253
254 return send_status;
255}
256
Yinghai Luc1eeb2d2009-02-16 23:02:14 -0800257void native_apic_icr_write(u32 low, u32 id)
Suresh Siddha1b374e42008-07-10 11:16:49 -0700258{
Jan Kiszkaea7bdc62014-01-27 20:14:06 +0100259 unsigned long flags;
260
261 local_irq_save(flags);
Cyrill Gorcunoved4e5ec2008-08-15 13:51:20 +0200262 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
Suresh Siddha1b374e42008-07-10 11:16:49 -0700263 apic_write(APIC_ICR, low);
Jan Kiszkaea7bdc62014-01-27 20:14:06 +0100264 local_irq_restore(flags);
Suresh Siddha1b374e42008-07-10 11:16:49 -0700265}
266
Yinghai Luc1eeb2d2009-02-16 23:02:14 -0800267u64 native_apic_icr_read(void)
Suresh Siddha1b374e42008-07-10 11:16:49 -0700268{
269 u32 icr1, icr2;
270
271 icr2 = apic_read(APIC_ICR2);
272 icr1 = apic_read(APIC_ICR);
273
Cyrill Gorcunovcf9768d72008-08-16 23:21:55 +0400274 return icr1 | ((u64)icr2 << 32);
Suresh Siddha1b374e42008-07-10 11:16:49 -0700275}
276
Cyrill Gorcunov7c37e482008-08-24 02:01:40 -0700277#ifdef CONFIG_X86_32
278/**
279 * get_physical_broadcast - Get number of physical broadcast IDs
280 */
281int get_physical_broadcast(void)
282{
283 return modern_apic() ? 0xff : 0xf;
284}
285#endif
286
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100287/**
288 * lapic_get_maxlvt - get the maximum number of local vector table entries
289 */
290int lapic_get_maxlvt(void)
291{
Cyrill Gorcunov36a028d2008-07-24 13:52:28 +0200292 unsigned int v;
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100293
294 v = apic_read(APIC_LVR);
Cyrill Gorcunov36a028d2008-07-24 13:52:28 +0200295 /*
296 * - we always have APIC integrated on 64bit mode
297 * - 82489DXs do not report # of LVT entries
298 */
299 return APIC_INTEGRATED(GET_APIC_VERSION(v)) ? GET_APIC_MAXLVT(v) : 2;
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100300}
301
302/*
Cyrill Gorcunov274cfe52008-08-16 23:21:53 +0400303 * Local APIC timer
304 */
305
Cyrill Gorcunovc40aaec2008-08-18 20:45:55 +0400306/* Clock divisor */
Cyrill Gorcunovc40aaec2008-08-18 20:45:55 +0400307#define APIC_DIVISOR 16
Suresh Siddha279f1462012-10-22 14:37:58 -0700308#define TSC_DIVISOR 32
Cyrill Gorcunovf07f4f92008-08-15 13:51:21 +0200309
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100310/*
311 * This function sets up the local APIC timer, with a timeout of
312 * 'clocks' APIC bus clock. During calibration we actually call
313 * this function twice on the boot CPU, once with a bogus timeout
314 * value, second time for real. The other (noncalibrating) CPUs
315 * call this function only once, with the real, calibrated value.
316 *
317 * We do reads before writes even if unnecessary, to get around the
318 * P5 APIC double write bug.
319 */
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100320static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
321{
322 unsigned int lvtt_value, tmp_value;
323
324 lvtt_value = LOCAL_TIMER_VECTOR;
325 if (!oneshot)
326 lvtt_value |= APIC_LVT_TIMER_PERIODIC;
Suresh Siddha279f1462012-10-22 14:37:58 -0700327 else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
328 lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE;
329
Cyrill Gorcunovf07f4f92008-08-15 13:51:21 +0200330 if (!lapic_is_integrated())
331 lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV);
332
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100333 if (!irqen)
334 lvtt_value |= APIC_LVT_MASKED;
335
336 apic_write(APIC_LVTT, lvtt_value);
337
Suresh Siddha279f1462012-10-22 14:37:58 -0700338 if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
339 printk_once(KERN_DEBUG "TSC deadline timer enabled\n");
340 return;
341 }
342
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100343 /*
344 * Divide PICLK by 16
345 */
346 tmp_value = apic_read(APIC_TDCR);
Cyrill Gorcunovc40aaec2008-08-18 20:45:55 +0400347 apic_write(APIC_TDCR,
348 (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
349 APIC_TDR_DIV_16);
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100350
351 if (!oneshot)
Cyrill Gorcunovf07f4f92008-08-15 13:51:21 +0200352 apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100353}
354
355/*
Robert Richtera68c4392010-10-06 12:27:53 +0200356 * Setup extended LVT, AMD specific
Robert Richter7b83dae2008-01-30 13:30:40 +0100357 *
Robert Richtera68c4392010-10-06 12:27:53 +0200358 * Software should use the LVT offsets the BIOS provides. The offsets
359 * are determined by the subsystems using it like those for MCE
360 * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
361 * are supported. Beginning with family 10h at least 4 offsets are
362 * available.
Robert Richter286f5712008-07-22 21:08:46 +0200363 *
Robert Richtera68c4392010-10-06 12:27:53 +0200364 * Since the offsets must be consistent for all cores, we keep track
365 * of the LVT offsets in software and reserve the offset for the same
366 * vector also to be used on other cores. An offset is freed by
367 * setting the entry to APIC_EILVT_MASKED.
368 *
369 * If the BIOS is right, there should be no conflicts. Otherwise a
370 * "[Firmware Bug]: ..." error message is generated. However, if
371 * software does not properly determines the offsets, it is not
372 * necessarily a BIOS bug.
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100373 */
Robert Richter7b83dae2008-01-30 13:30:40 +0100374
Robert Richtera68c4392010-10-06 12:27:53 +0200375static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100376
Robert Richtera68c4392010-10-06 12:27:53 +0200377static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
378{
379 return (old & APIC_EILVT_MASKED)
380 || (new == APIC_EILVT_MASKED)
381 || ((new & ~APIC_EILVT_MASKED) == old);
382}
383
384static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
385{
Robert Richter8abc3122012-03-27 20:04:02 +0200386 unsigned int rsvd, vector;
Robert Richtera68c4392010-10-06 12:27:53 +0200387
388 if (offset >= APIC_EILVT_NR_MAX)
389 return ~0;
390
Robert Richter8abc3122012-03-27 20:04:02 +0200391 rsvd = atomic_read(&eilvt_offsets[offset]);
Robert Richtera68c4392010-10-06 12:27:53 +0200392 do {
Robert Richter8abc3122012-03-27 20:04:02 +0200393 vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */
394 if (vector && !eilvt_entry_is_changeable(vector, new))
Robert Richtera68c4392010-10-06 12:27:53 +0200395 /* may not change if vectors are different */
396 return rsvd;
397 rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new);
398 } while (rsvd != new);
399
Robert Richter8abc3122012-03-27 20:04:02 +0200400 rsvd &= ~APIC_EILVT_MASKED;
401 if (rsvd && rsvd != vector)
402 pr_info("LVT offset %d assigned for vector 0x%02x\n",
403 offset, rsvd);
404
Robert Richtera68c4392010-10-06 12:27:53 +0200405 return new;
406}
407
408/*
409 * If mask=1, the LVT entry does not generate interrupts while mask=0
Robert Richtercbf74ce2011-05-30 16:31:11 +0200410 * enables the vector. See also the BKDGs. Must be called with
411 * preemption disabled.
Robert Richtera68c4392010-10-06 12:27:53 +0200412 */
413
Robert Richter27afdf22010-10-06 12:27:54 +0200414int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
Robert Richtera68c4392010-10-06 12:27:53 +0200415{
416 unsigned long reg = APIC_EILVTn(offset);
417 unsigned int new, old, reserved;
418
419 new = (mask << 16) | (msg_type << 8) | vector;
420 old = apic_read(reg);
421 reserved = reserve_eilvt_offset(offset, new);
422
423 if (reserved != new) {
Robert Richtereb48c9c2010-10-25 16:03:39 +0200424 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
425 "vector 0x%x, but the register is already in use for "
426 "vector 0x%x on another cpu\n",
427 smp_processor_id(), reg, offset, new, reserved);
Robert Richtera68c4392010-10-06 12:27:53 +0200428 return -EINVAL;
429 }
430
431 if (!eilvt_entry_is_changeable(old, new)) {
Robert Richtereb48c9c2010-10-25 16:03:39 +0200432 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
433 "vector 0x%x, but the register is already in use for "
434 "vector 0x%x on this cpu\n",
435 smp_processor_id(), reg, offset, new, old);
Robert Richtera68c4392010-10-06 12:27:53 +0200436 return -EBUSY;
437 }
438
439 apic_write(reg, new);
440
441 return 0;
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100442}
Robert Richter27afdf22010-10-06 12:27:54 +0200443EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
Robert Richter7b83dae2008-01-30 13:30:40 +0100444
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100445/*
446 * Program the next event, relative to now
447 */
Thomas Gleixnerba7eda42007-10-12 23:04:07 +0200448static int lapic_next_event(unsigned long delta,
449 struct clock_event_device *evt)
450{
451 apic_write(APIC_TMICT, delta);
452 return 0;
453}
454
Suresh Siddha279f1462012-10-22 14:37:58 -0700455static int lapic_next_deadline(unsigned long delta,
456 struct clock_event_device *evt)
457{
458 u64 tsc;
459
460 rdtscll(tsc);
461 wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
462 return 0;
463}
464
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100465/*
466 * Setup the lapic timer in periodic or oneshot mode
467 */
Thomas Gleixnerba7eda42007-10-12 23:04:07 +0200468static void lapic_timer_setup(enum clock_event_mode mode,
469 struct clock_event_device *evt)
470{
471 unsigned long flags;
472 unsigned int v;
473
474 /* Lapic used as dummy for broadcast ? */
475 if (evt->features & CLOCK_EVT_FEAT_DUMMY)
476 return;
477
478 local_irq_save(flags);
479
480 switch (mode) {
481 case CLOCK_EVT_MODE_PERIODIC:
482 case CLOCK_EVT_MODE_ONESHOT:
Jacob Pan1ade93e2011-11-10 13:42:40 +0000483 __setup_APIC_LVTT(lapic_timer_frequency,
Thomas Gleixnerba7eda42007-10-12 23:04:07 +0200484 mode != CLOCK_EVT_MODE_PERIODIC, 1);
485 break;
486 case CLOCK_EVT_MODE_UNUSED:
487 case CLOCK_EVT_MODE_SHUTDOWN:
488 v = apic_read(APIC_LVTT);
489 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
490 apic_write(APIC_LVTT, v);
Andreas Herrmann6f9b4102009-10-27 11:01:38 +0100491 apic_write(APIC_TMICT, 0);
Thomas Gleixnerba7eda42007-10-12 23:04:07 +0200492 break;
493 case CLOCK_EVT_MODE_RESUME:
494 /* Nothing to do here */
495 break;
496 }
497
498 local_irq_restore(flags);
499}
500
501/*
502 * Local APIC timer broadcast function
503 */
Mike Travis96289372008-12-31 18:08:46 -0800504static void lapic_timer_broadcast(const struct cpumask *mask)
Thomas Gleixnerba7eda42007-10-12 23:04:07 +0200505{
506#ifdef CONFIG_SMP
Ingo Molnardac5f412009-01-28 15:42:24 +0100507 apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
Thomas Gleixnerba7eda42007-10-12 23:04:07 +0200508#endif
509}
510
Henrik Kretzschmar25874a22011-03-11 08:02:36 +0100511
512/*
513 * The local apic timer can be used for any function which is CPU local.
514 */
515static struct clock_event_device lapic_clockevent = {
516 .name = "lapic",
517 .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT
518 | CLOCK_EVT_FEAT_C3STOP | CLOCK_EVT_FEAT_DUMMY,
519 .shift = 32,
520 .set_mode = lapic_timer_setup,
521 .set_next_event = lapic_next_event,
522 .broadcast = lapic_timer_broadcast,
523 .rating = 100,
524 .irq = -1,
525};
526static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
527
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100528/*
Uwe Kleine-König421f91d2010-06-11 12:17:00 +0200529 * Setup the local APIC timer for this CPU. Copy the initialized values
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100530 * of the boot CPU and register the clock event in the framework.
531 */
Paul Gortmaker148f9bb2013-06-18 18:23:59 -0400532static void setup_APIC_timer(void)
Fernando Luis VazquezCao8339e9f2007-05-02 19:27:17 +0200533{
Christoph Lameter89cbc762014-08-17 12:30:40 -0500534 struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100535
Christoph Lameter349c0042011-03-12 12:50:10 +0100536 if (this_cpu_has(X86_FEATURE_ARAT)) {
Venkatesh Pallipadidb954b52009-04-06 18:51:29 -0700537 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
538 /* Make LAPIC timer preferrable over percpu HPET */
539 lapic_clockevent.rating = 150;
540 }
541
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100542 memcpy(levt, &lapic_clockevent, sizeof(*levt));
Rusty Russell320ab2b2008-12-13 21:20:26 +1030543 levt->cpumask = cpumask_of(smp_processor_id());
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100544
Suresh Siddha279f1462012-10-22 14:37:58 -0700545 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
546 levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
547 CLOCK_EVT_FEAT_DUMMY);
548 levt->set_next_event = lapic_next_deadline;
549 clockevents_config_and_register(levt,
550 (tsc_khz / TSC_DIVISOR) * 1000,
551 0xF, ~0UL);
552 } else
553 clockevents_register_device(levt);
Fernando Luis VazquezCao8339e9f2007-05-02 19:27:17 +0200554}
555
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700556/*
557 * In this functions we calibrate APIC bus clocks to the external timer.
558 *
559 * We want to do the calibration only once since we want to have local timer
560 * irqs syncron. CPUs connected by the same APIC bus have the very same bus
561 * frequency.
562 *
563 * This was previously done by reading the PIT/HPET and waiting for a wrap
564 * around to find out, that a tick has elapsed. I have a box, where the PIT
565 * readout is broken, so it never gets out of the wait loop again. This was
566 * also reported by others.
567 *
568 * Monitoring the jiffies value is inaccurate and the clockevents
569 * infrastructure allows us to do a simple substitution of the interrupt
570 * handler.
571 *
572 * The calibration routine also uses the pm_timer when possible, as the PIT
573 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
574 * back to normal later in the boot process).
575 */
576
577#define LAPIC_CAL_LOOPS (HZ/10)
578
579static __initdata int lapic_cal_loops = -1;
580static __initdata long lapic_cal_t1, lapic_cal_t2;
581static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
582static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
583static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
584
585/*
586 * Temporary interrupt handler.
587 */
588static void __init lapic_cal_handler(struct clock_event_device *dev)
589{
590 unsigned long long tsc = 0;
591 long tapic = apic_read(APIC_TMCCT);
592 unsigned long pm = acpi_pm_read_early();
593
594 if (cpu_has_tsc)
595 rdtscll(tsc);
596
597 switch (lapic_cal_loops++) {
598 case 0:
599 lapic_cal_t1 = tapic;
600 lapic_cal_tsc1 = tsc;
601 lapic_cal_pm1 = pm;
602 lapic_cal_j1 = jiffies;
603 break;
604
605 case LAPIC_CAL_LOOPS:
606 lapic_cal_t2 = tapic;
607 lapic_cal_tsc2 = tsc;
608 if (pm < lapic_cal_pm1)
609 pm += ACPI_PM_OVRRUN;
610 lapic_cal_pm2 = pm;
611 lapic_cal_j2 = jiffies;
612 break;
613 }
614}
615
Yasuaki Ishimatsu754ef0c2009-01-28 12:51:09 +0900616static int __init
617calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
Cyrill Gorcunovb1898922008-09-12 23:58:24 +0400618{
619 const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
620 const long pm_thresh = pm_100ms / 100;
621 unsigned long mult;
622 u64 res;
623
624#ifndef CONFIG_X86_PM_TIMER
625 return -1;
626#endif
627
Yasuaki Ishimatsu39ba5d42009-01-28 12:52:24 +0900628 apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
Cyrill Gorcunovb1898922008-09-12 23:58:24 +0400629
630 /* Check, if the PM timer is available */
631 if (!deltapm)
632 return -1;
633
634 mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
635
636 if (deltapm > (pm_100ms - pm_thresh) &&
637 deltapm < (pm_100ms + pm_thresh)) {
Yasuaki Ishimatsu39ba5d42009-01-28 12:52:24 +0900638 apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
Yasuaki Ishimatsu754ef0c2009-01-28 12:51:09 +0900639 return 0;
640 }
641
642 res = (((u64)deltapm) * mult) >> 22;
643 do_div(res, 1000000);
644 pr_warning("APIC calibration not consistent "
Yasuaki Ishimatsu39ba5d42009-01-28 12:52:24 +0900645 "with PM-Timer: %ldms instead of 100ms\n",(long)res);
Yasuaki Ishimatsu754ef0c2009-01-28 12:51:09 +0900646
647 /* Correct the lapic counter value */
648 res = (((u64)(*delta)) * pm_100ms);
649 do_div(res, deltapm);
650 pr_info("APIC delta adjusted to PM-Timer: "
651 "%lu (%ld)\n", (unsigned long)res, *delta);
652 *delta = (long)res;
653
654 /* Correct the tsc counter value */
655 if (cpu_has_tsc) {
656 res = (((u64)(*deltatsc)) * pm_100ms);
Cyrill Gorcunovb1898922008-09-12 23:58:24 +0400657 do_div(res, deltapm);
Yasuaki Ishimatsu754ef0c2009-01-28 12:51:09 +0900658 apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
Frans Pop3235dc32010-02-06 18:47:17 +0100659 "PM-Timer: %lu (%ld)\n",
Yasuaki Ishimatsu754ef0c2009-01-28 12:51:09 +0900660 (unsigned long)res, *deltatsc);
661 *deltatsc = (long)res;
Cyrill Gorcunovb1898922008-09-12 23:58:24 +0400662 }
663
664 return 0;
665}
666
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700667static int __init calibrate_APIC_clock(void)
668{
Christoph Lameter89cbc762014-08-17 12:30:40 -0500669 struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700670 void (*real_handler)(struct clock_event_device *dev);
671 unsigned long deltaj;
Yasuaki Ishimatsu754ef0c2009-01-28 12:51:09 +0900672 long delta, deltatsc;
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700673 int pm_referenced = 0;
674
Jacob Pan1ade93e2011-11-10 13:42:40 +0000675 /**
676 * check if lapic timer has already been calibrated by platform
677 * specific routine, such as tsc calibration code. if so, we just fill
678 * in the clockevent structure and return.
679 */
680
Suresh Siddha279f1462012-10-22 14:37:58 -0700681 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
682 return 0;
683 } else if (lapic_timer_frequency) {
Jacob Pan1ade93e2011-11-10 13:42:40 +0000684 apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n",
685 lapic_timer_frequency);
686 lapic_clockevent.mult = div_sc(lapic_timer_frequency/APIC_DIVISOR,
687 TICK_NSEC, lapic_clockevent.shift);
688 lapic_clockevent.max_delta_ns =
689 clockevent_delta2ns(0x7FFFFF, &lapic_clockevent);
690 lapic_clockevent.min_delta_ns =
691 clockevent_delta2ns(0xF, &lapic_clockevent);
692 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
693 return 0;
694 }
695
Suresh Siddha279f1462012-10-22 14:37:58 -0700696 apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
697 "calibrating APIC timer ...\n");
698
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700699 local_irq_disable();
700
701 /* Replace the global interrupt handler */
702 real_handler = global_clock_event->event_handler;
703 global_clock_event->event_handler = lapic_cal_handler;
704
705 /*
Cyrill Gorcunov81608f32008-10-10 19:00:17 +0400706 * Setup the APIC counter to maximum. There is no way the lapic
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700707 * can underflow in the 100ms detection time frame
708 */
Cyrill Gorcunov81608f32008-10-10 19:00:17 +0400709 __setup_APIC_LVTT(0xffffffff, 0, 0);
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700710
711 /* Let the interrupts run */
712 local_irq_enable();
713
714 while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
715 cpu_relax();
716
717 local_irq_disable();
718
719 /* Restore the real event handler */
720 global_clock_event->event_handler = real_handler;
721
722 /* Build delta t1-t2 as apic timer counts down */
723 delta = lapic_cal_t1 - lapic_cal_t2;
724 apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
725
Yasuaki Ishimatsu754ef0c2009-01-28 12:51:09 +0900726 deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
727
Cyrill Gorcunovb1898922008-09-12 23:58:24 +0400728 /* we trust the PM based calibration if possible */
729 pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
Yasuaki Ishimatsu754ef0c2009-01-28 12:51:09 +0900730 &delta, &deltatsc);
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700731
732 /* Calculate the scaled math multiplication factor */
733 lapic_clockevent.mult = div_sc(delta, TICK_NSEC * LAPIC_CAL_LOOPS,
734 lapic_clockevent.shift);
735 lapic_clockevent.max_delta_ns =
Pierre Tardy4aed89d2011-01-06 16:23:29 +0100736 clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700737 lapic_clockevent.min_delta_ns =
738 clockevent_delta2ns(0xF, &lapic_clockevent);
739
Jacob Pan1ade93e2011-11-10 13:42:40 +0000740 lapic_timer_frequency = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700741
742 apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
Thomas Gleixner411462f2009-11-16 11:52:39 +0100743 apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700744 apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
Jacob Pan1ade93e2011-11-10 13:42:40 +0000745 lapic_timer_frequency);
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700746
747 if (cpu_has_tsc) {
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700748 apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
749 "%ld.%04ld MHz.\n",
Yasuaki Ishimatsu754ef0c2009-01-28 12:51:09 +0900750 (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
751 (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700752 }
753
754 apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
755 "%u.%04u MHz.\n",
Jacob Pan1ade93e2011-11-10 13:42:40 +0000756 lapic_timer_frequency / (1000000 / HZ),
757 lapic_timer_frequency % (1000000 / HZ));
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700758
759 /*
760 * Do a sanity check on the APIC calibration result
761 */
Jacob Pan1ade93e2011-11-10 13:42:40 +0000762 if (lapic_timer_frequency < (1000000 / HZ)) {
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700763 local_irq_enable();
Cyrill Gorcunovba21ebb2008-11-10 09:16:41 +0100764 pr_warning("APIC frequency too slow, disabling apic timer\n");
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700765 return -1;
766 }
767
768 levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
769
Cyrill Gorcunovb1898922008-09-12 23:58:24 +0400770 /*
771 * PM timer calibration failed or not turned on
772 * so lets try APIC timer based calibration
773 */
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700774 if (!pm_referenced) {
775 apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
776
777 /*
778 * Setup the apic timer manually
779 */
780 levt->event_handler = lapic_cal_handler;
781 lapic_timer_setup(CLOCK_EVT_MODE_PERIODIC, levt);
782 lapic_cal_loops = -1;
783
784 /* Let the interrupts run */
785 local_irq_enable();
786
787 while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
788 cpu_relax();
789
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700790 /* Stop the lapic timer */
791 lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, levt);
792
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700793 /* Jiffies delta */
794 deltaj = lapic_cal_j2 - lapic_cal_j1;
795 apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
796
797 /* Check, if the jiffies result is consistent */
798 if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
799 apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
800 else
801 levt->features |= CLOCK_EVT_FEAT_DUMMY;
802 } else
803 local_irq_enable();
804
805 if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
Jaswinder Singh Rajpute423e332009-01-04 16:16:25 +0530806 pr_warning("APIC timer disabled due to verification failure\n");
Yinghai Lu2f04fa82008-08-24 02:01:54 -0700807 return -1;
808 }
809
810 return 0;
811}
812
Hiroshi Shimamotoe83a5fd2008-01-30 13:32:35 +0100813/*
814 * Setup the boot APIC
815 *
816 * Calibrate and verify the result.
817 */
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100818void __init setup_boot_APIC_clock(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819{
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100820 /*
Cyrill Gorcunov274cfe52008-08-16 23:21:53 +0400821 * The local apic timer can be disabled via the kernel
822 * commandline or from the CPU detection code. Register the lapic
823 * timer as a dummy clock event source on SMP systems, so the
824 * broadcast mechanism is used. On UP systems simply ignore it.
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100825 */
826 if (disable_apic_timer) {
Cyrill Gorcunovba21ebb2008-11-10 09:16:41 +0100827 pr_info("Disabling APIC timer\n");
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100828 /* No broadcast on UP ! */
Thomas Gleixner9d099512008-01-30 13:33:04 +0100829 if (num_possible_cpus() > 1) {
830 lapic_clockevent.mult = 1;
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100831 setup_APIC_timer();
Thomas Gleixner9d099512008-01-30 13:33:04 +0100832 }
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100833 return;
834 }
Thomas Gleixner6935d1f2007-07-21 17:10:17 +0200835
Cyrill Gorcunov89b3b1f42008-07-15 21:02:54 +0400836 if (calibrate_APIC_clock()) {
Thomas Gleixnerc2b84b32008-01-30 13:33:04 +0100837 /* No broadcast on UP ! */
838 if (num_possible_cpus() > 1)
839 setup_APIC_timer();
840 return;
841 }
842
843 /*
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100844 * If nmi_watchdog is set to IO_APIC, we need the
845 * PIT/HPET going. Otherwise register lapic as a dummy
846 * device.
847 */
Don Zickus072b1982010-11-12 11:22:24 -0500848 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100849
Cyrill Gorcunov274cfe52008-08-16 23:21:53 +0400850 /* Setup the lapic or request the broadcast */
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100851 setup_APIC_timer();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852}
853
Paul Gortmaker148f9bb2013-06-18 18:23:59 -0400854void setup_secondary_APIC_clock(void)
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100855{
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100856 setup_APIC_timer();
857}
858
859/*
860 * The guts of the apic timer interrupt
861 */
862static void local_apic_timer_interrupt(void)
863{
864 int cpu = smp_processor_id();
865 struct clock_event_device *evt = &per_cpu(lapic_events, cpu);
866
867 /*
868 * Normally we should not be here till LAPIC has been initialized but
869 * in some cases like kdump, its possible that there is a pending LAPIC
870 * timer interrupt from previous kernel's context and is delivered in
871 * new kernel the moment interrupts are enabled.
872 *
873 * Interrupts are enabled early and LAPIC is setup much later, hence
874 * its possible that when we get here evt->event_handler is NULL.
875 * Check for event_handler being NULL and discard the interrupt as
876 * spurious.
877 */
878 if (!evt->event_handler) {
Cyrill Gorcunovba21ebb2008-11-10 09:16:41 +0100879 pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu);
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100880 /* Switch it off */
881 lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, evt);
882 return;
883 }
884
885 /*
886 * the NMI deadlock-detector uses this.
887 */
Hiroshi Shimamoto915b0d02008-12-08 19:19:26 -0800888 inc_irq_stat(apic_timer_irqs);
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100889
890 evt->event_handler(evt);
891}
892
893/*
894 * Local APIC timer interrupt. This is the most natural way for doing
895 * local interrupts, but local timer interrupts can be emulated by
896 * broadcast interrupts too. [in case the hw doesn't support APIC timers]
897 *
898 * [ if a single-CPU system runs an SMP kernel then we call the local
899 * interrupt as well. Thus we cannot inline the local irq ... ]
900 */
Andi Kleen1d9090e2013-08-05 15:02:37 -0700901__visible void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100902{
903 struct pt_regs *old_regs = set_irq_regs(regs);
904
905 /*
906 * NOTE! We'd better ACK the irq immediately,
907 * because timer handling can be slow.
Seiji Aguchieddc0e92013-06-20 11:45:17 -0400908 *
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100909 * update_process_times() expects us to have done irq_enter().
910 * Besides, if we don't timer interrupts ignore the global
911 * interrupt lock, which is the WrongThing (tm) to do.
912 */
Seiji Aguchieddc0e92013-06-20 11:45:17 -0400913 entering_ack_irq();
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100914 local_apic_timer_interrupt();
Seiji Aguchieddc0e92013-06-20 11:45:17 -0400915 exiting_irq();
Cyrill Gorcunov274cfe52008-08-16 23:21:53 +0400916
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100917 set_irq_regs(old_regs);
918}
919
Andi Kleen1d9090e2013-08-05 15:02:37 -0700920__visible void __irq_entry smp_trace_apic_timer_interrupt(struct pt_regs *regs)
Seiji Aguchicf910e82013-06-20 11:46:53 -0400921{
922 struct pt_regs *old_regs = set_irq_regs(regs);
923
924 /*
925 * NOTE! We'd better ACK the irq immediately,
926 * because timer handling can be slow.
927 *
928 * update_process_times() expects us to have done irq_enter().
929 * Besides, if we don't timer interrupts ignore the global
930 * interrupt lock, which is the WrongThing (tm) to do.
931 */
932 entering_ack_irq();
933 trace_local_timer_entry(LOCAL_TIMER_VECTOR);
934 local_apic_timer_interrupt();
935 trace_local_timer_exit(LOCAL_TIMER_VECTOR);
936 exiting_irq();
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100937
938 set_irq_regs(old_regs);
939}
940
941int setup_profiling_timer(unsigned int multiplier)
942{
943 return -EINVAL;
944}
945
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100946/*
947 * Local APIC start and shutdown
948 */
949
950/**
951 * clear_local_APIC - shutdown the local APIC
952 *
953 * This is called, when a CPU is disabled and before rebooting, so the state of
954 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
955 * leftovers during boot.
956 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957void clear_local_APIC(void)
958{
Chuck Ebbert2584a822008-05-20 18:18:12 -0400959 int maxlvt;
Thomas Gleixner0e078e22008-01-30 13:30:20 +0100960 u32 v;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961
Andi Kleend3432892008-01-30 13:33:17 +0100962 /* APIC hasn't been mapped yet */
Suresh Siddhafc1edaf2009-04-20 13:02:27 -0700963 if (!x2apic_mode && !apic_phys)
Andi Kleend3432892008-01-30 13:33:17 +0100964 return;
965
966 maxlvt = lapic_get_maxlvt();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967 /*
Siddha, Suresh B704fc592006-06-26 13:59:53 +0200968 * Masking an LVT entry can trigger a local APIC error
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 * if the vector is zero. Mask LVTERR first to prevent this.
970 */
971 if (maxlvt >= 3) {
972 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
Andi Kleen11a8e772006-01-11 22:46:51 +0100973 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974 }
975 /*
976 * Careful: we have to set masks only first to deassert
977 * any level-triggered sources.
978 */
979 v = apic_read(APIC_LVTT);
Andi Kleen11a8e772006-01-11 22:46:51 +0100980 apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981 v = apic_read(APIC_LVT0);
Andi Kleen11a8e772006-01-11 22:46:51 +0100982 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 v = apic_read(APIC_LVT1);
Andi Kleen11a8e772006-01-11 22:46:51 +0100984 apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985 if (maxlvt >= 4) {
986 v = apic_read(APIC_LVTPC);
Andi Kleen11a8e772006-01-11 22:46:51 +0100987 apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 }
989
Cyrill Gorcunov67640142008-08-16 23:21:50 +0400990 /* lets not touch this if we didn't frob it */
Andi Kleen4efc0672009-04-28 19:07:31 +0200991#ifdef CONFIG_X86_THERMAL_VECTOR
Cyrill Gorcunov67640142008-08-16 23:21:50 +0400992 if (maxlvt >= 5) {
993 v = apic_read(APIC_LVTTHMR);
994 apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
995 }
996#endif
Andi Kleen5ca86812009-02-12 13:49:37 +0100997#ifdef CONFIG_X86_MCE_INTEL
998 if (maxlvt >= 6) {
999 v = apic_read(APIC_LVTCMCI);
1000 if (!(v & APIC_LVT_MASKED))
1001 apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
1002 }
1003#endif
1004
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 /*
1006 * Clean APIC state for other OSs:
1007 */
Andi Kleen11a8e772006-01-11 22:46:51 +01001008 apic_write(APIC_LVTT, APIC_LVT_MASKED);
1009 apic_write(APIC_LVT0, APIC_LVT_MASKED);
1010 apic_write(APIC_LVT1, APIC_LVT_MASKED);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011 if (maxlvt >= 3)
Andi Kleen11a8e772006-01-11 22:46:51 +01001012 apic_write(APIC_LVTERR, APIC_LVT_MASKED);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013 if (maxlvt >= 4)
Andi Kleen11a8e772006-01-11 22:46:51 +01001014 apic_write(APIC_LVTPC, APIC_LVT_MASKED);
Cyrill Gorcunov67640142008-08-16 23:21:50 +04001015
1016 /* Integrated APIC (!82489DX) ? */
1017 if (lapic_is_integrated()) {
1018 if (maxlvt > 3)
1019 /* Clear ESR due to Pentium errata 3AP and 11AP */
1020 apic_write(APIC_ESR, 0);
1021 apic_read(APIC_ESR);
1022 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023}
1024
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001025/**
1026 * disable_local_APIC - clear and disable the local APIC
1027 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028void disable_local_APIC(void)
1029{
1030 unsigned int value;
1031
Jan Beulich4a13ad02009-01-14 12:28:51 +00001032 /* APIC hasn't been mapped yet */
Yinghai Lufd19dce2010-07-15 00:00:59 -07001033 if (!x2apic_mode && !apic_phys)
Jan Beulich4a13ad02009-01-14 12:28:51 +00001034 return;
1035
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 clear_local_APIC();
1037
1038 /*
1039 * Disable APIC (implies clearing of registers
1040 * for 82489DX!).
1041 */
1042 value = apic_read(APIC_SPIV);
1043 value &= ~APIC_SPIV_APIC_ENABLED;
Andi Kleen11a8e772006-01-11 22:46:51 +01001044 apic_write(APIC_SPIV, value);
Cyrill Gorcunov990b1832008-08-18 20:45:51 +04001045
1046#ifdef CONFIG_X86_32
1047 /*
1048 * When LAPIC was disabled by the BIOS and enabled by the kernel,
1049 * restore the disabled state.
1050 */
1051 if (enabled_via_apicbase) {
1052 unsigned int l, h;
1053
1054 rdmsr(MSR_IA32_APICBASE, l, h);
1055 l &= ~MSR_IA32_APICBASE_ENABLE;
1056 wrmsr(MSR_IA32_APICBASE, l, h);
1057 }
1058#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001059}
1060
Cyrill Gorcunovfe4024d2008-08-18 20:45:52 +04001061/*
1062 * If Linux enabled the LAPIC against the BIOS default disable it down before
1063 * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
1064 * not power-off. Additionally clear all LVT entries before disable_local_APIC
1065 * for the case where Linux didn't enable the LAPIC.
1066 */
Hiroshi Shimamoto9b7711f2007-10-19 18:21:11 -07001067void lapic_shutdown(void)
1068{
1069 unsigned long flags;
1070
Cyrill Gorcunov83121362009-09-15 11:12:30 +04001071 if (!cpu_has_apic && !apic_from_smp_config())
Hiroshi Shimamoto9b7711f2007-10-19 18:21:11 -07001072 return;
1073
1074 local_irq_save(flags);
1075
Cyrill Gorcunovfe4024d2008-08-18 20:45:52 +04001076#ifdef CONFIG_X86_32
1077 if (!enabled_via_apicbase)
1078 clear_local_APIC();
1079 else
1080#endif
1081 disable_local_APIC();
1082
Hiroshi Shimamoto9b7711f2007-10-19 18:21:11 -07001083
1084 local_irq_restore(flags);
1085}
1086
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001087/**
1088 * sync_Arb_IDs - synchronize APIC bus arbitration IDs
1089 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090void __init sync_Arb_IDs(void)
1091{
Cyrill Gorcunov296cb952008-08-15 13:51:23 +02001092 /*
1093 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1094 * needed on AMD.
1095 */
1096 if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001097 return;
1098
1099 /*
1100 * Wait for idle.
1101 */
1102 apic_wait_icr_idle();
1103
1104 apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
Cyrill Gorcunov6f6da972008-08-15 23:05:19 +04001105 apic_write(APIC_ICR, APIC_DEST_ALLINC |
1106 APIC_INT_LEVELTRIG | APIC_DM_INIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107}
1108
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109/*
1110 * An initial setup of the virtual wire mode.
1111 */
1112void __init init_bsp_APIC(void)
1113{
Andi Kleen11a8e772006-01-11 22:46:51 +01001114 unsigned int value;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115
1116 /*
1117 * Don't do the setup now if we have a SMP BIOS as the
1118 * through-I/O-APIC virtual wire mode might be active.
1119 */
1120 if (smp_found_config || !cpu_has_apic)
1121 return;
1122
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123 /*
1124 * Do not trust the local APIC being empty at bootup.
1125 */
1126 clear_local_APIC();
1127
1128 /*
1129 * Enable APIC.
1130 */
1131 value = apic_read(APIC_SPIV);
1132 value &= ~APIC_VECTOR_MASK;
1133 value |= APIC_SPIV_APIC_ENABLED;
Cyrill Gorcunov638c0412008-08-15 23:05:18 +04001134
1135#ifdef CONFIG_X86_32
1136 /* This bit is reserved on P4/Xeon and should be cleared */
1137 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1138 (boot_cpu_data.x86 == 15))
1139 value &= ~APIC_SPIV_FOCUS_DISABLED;
1140 else
1141#endif
1142 value |= APIC_SPIV_FOCUS_DISABLED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143 value |= SPURIOUS_APIC_VECTOR;
Andi Kleen11a8e772006-01-11 22:46:51 +01001144 apic_write(APIC_SPIV, value);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145
1146 /*
1147 * Set up the virtual wire mode.
1148 */
Andi Kleen11a8e772006-01-11 22:46:51 +01001149 apic_write(APIC_LVT0, APIC_DM_EXTINT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150 value = APIC_DM_NMI;
Cyrill Gorcunov638c0412008-08-15 23:05:18 +04001151 if (!lapic_is_integrated()) /* 82489DX */
1152 value |= APIC_LVT_LEVEL_TRIGGER;
Andi Kleen11a8e772006-01-11 22:46:51 +01001153 apic_write(APIC_LVT1, value);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154}
1155
Paul Gortmaker148f9bb2013-06-18 18:23:59 -04001156static void lapic_setup_esr(void)
Cyrill Gorcunovc43da2f2008-08-18 20:45:54 +04001157{
Cyrill Gorcunov9df08f12008-09-14 11:55:37 +04001158 unsigned int oldvalue, value, maxlvt;
Cyrill Gorcunovc43da2f2008-08-18 20:45:54 +04001159
Cyrill Gorcunov9df08f12008-09-14 11:55:37 +04001160 if (!lapic_is_integrated()) {
Cyrill Gorcunovba21ebb2008-11-10 09:16:41 +01001161 pr_info("No ESR for 82489DX.\n");
Cyrill Gorcunov9df08f12008-09-14 11:55:37 +04001162 return;
Cyrill Gorcunovc43da2f2008-08-18 20:45:54 +04001163 }
Cyrill Gorcunov9df08f12008-09-14 11:55:37 +04001164
Ingo Molnar08125d32009-01-28 05:08:44 +01001165 if (apic->disable_esr) {
Cyrill Gorcunov9df08f12008-09-14 11:55:37 +04001166 /*
1167 * Something untraceable is creating bad interrupts on
1168 * secondary quads ... for the moment, just leave the
1169 * ESR disabled - we can't do anything useful with the
1170 * errors anyway - mbligh
1171 */
Cyrill Gorcunovba21ebb2008-11-10 09:16:41 +01001172 pr_info("Leaving ESR disabled.\n");
Cyrill Gorcunov9df08f12008-09-14 11:55:37 +04001173 return;
1174 }
1175
1176 maxlvt = lapic_get_maxlvt();
1177 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1178 apic_write(APIC_ESR, 0);
1179 oldvalue = apic_read(APIC_ESR);
1180
1181 /* enables sending errors */
1182 value = ERROR_APIC_VECTOR;
1183 apic_write(APIC_LVTERR, value);
1184
1185 /*
1186 * spec says clear errors after enabling vector.
1187 */
1188 if (maxlvt > 3)
1189 apic_write(APIC_ESR, 0);
1190 value = apic_read(APIC_ESR);
1191 if (value != oldvalue)
1192 apic_printk(APIC_VERBOSE, "ESR value before enabling "
1193 "vector: 0x%08x after: 0x%08x\n",
1194 oldvalue, value);
Cyrill Gorcunovc43da2f2008-08-18 20:45:54 +04001195}
1196
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001197/**
1198 * setup_local_APIC - setup the local APIC
Tejun Heo0aa002f2010-12-09 11:47:21 +01001199 *
1200 * Used to setup local APIC while initializing BSP or bringin up APs.
1201 * Always called with preemption disabled.
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001202 */
Paul Gortmaker148f9bb2013-06-18 18:23:59 -04001203void setup_local_APIC(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204{
Tejun Heo0aa002f2010-12-09 11:47:21 +01001205 int cpu = smp_processor_id();
Kerstin Jonsson8c3ba8d2010-05-24 12:13:15 -07001206 unsigned int value, queued;
1207 int i, j, acked = 0;
1208 unsigned long long tsc = 0, ntsc;
Andy Lutomirskib47dcbd2014-10-15 10:12:07 -07001209 long long max_loops = cpu_khz ? cpu_khz : 1000000;
Kerstin Jonsson8c3ba8d2010-05-24 12:13:15 -07001210
1211 if (cpu_has_tsc)
1212 rdtscll(tsc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213
Jan Beulichf1182632009-01-14 12:27:35 +00001214 if (disable_apic) {
Henrik Kretzschmar7167d082011-02-22 15:38:05 +01001215 disable_ioapic_support();
Jan Beulichf1182632009-01-14 12:27:35 +00001216 return;
1217 }
1218
Cyrill Gorcunov89c38c22008-08-24 02:01:43 -07001219#ifdef CONFIG_X86_32
1220 /* Pound the ESR really hard over the head with a big hammer - mbligh */
Ingo Molnar08125d32009-01-28 05:08:44 +01001221 if (lapic_is_integrated() && apic->disable_esr) {
Cyrill Gorcunov89c38c22008-08-24 02:01:43 -07001222 apic_write(APIC_ESR, 0);
1223 apic_write(APIC_ESR, 0);
1224 apic_write(APIC_ESR, 0);
1225 apic_write(APIC_ESR, 0);
1226 }
1227#endif
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001228 perf_events_lapic_init();
Cyrill Gorcunov89c38c22008-08-24 02:01:43 -07001229
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230 /*
1231 * Double-check whether this APIC is really registered.
1232 * This is meaningless in clustered apic mode, so we skip it.
1233 */
Daniel Walkerc2777f92009-09-12 10:40:20 -07001234 BUG_ON(!apic->apic_id_registered());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001235
1236 /*
1237 * Intel recommends to set DFR, LDR and TPR before enabling
1238 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
1239 * document number 292116). So here it goes...
1240 */
Ingo Molnara5c43292009-01-28 06:50:47 +01001241 apic->init_apic_ldr();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242
Tejun Heo6f802c42011-01-23 14:37:31 +01001243#ifdef CONFIG_X86_32
1244 /*
Tejun Heoacb8bc02011-01-23 14:37:33 +01001245 * APIC LDR is initialized. If logical_apicid mapping was
1246 * initialized during get_smp_config(), make sure it matches the
1247 * actual value.
Tejun Heo6f802c42011-01-23 14:37:31 +01001248 */
Tejun Heoacb8bc02011-01-23 14:37:33 +01001249 i = early_per_cpu(x86_cpu_to_logical_apicid, cpu);
1250 WARN_ON(i != BAD_APICID && i != logical_smp_processor_id());
1251 /* always use the value from LDR */
Tejun Heo6f802c42011-01-23 14:37:31 +01001252 early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
1253 logical_smp_processor_id();
1254#endif
1255
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256 /*
1257 * Set Task Priority to 'accept all'. We never change this
1258 * later on.
1259 */
1260 value = apic_read(APIC_TASKPRI);
1261 value &= ~APIC_TPRI_MASK;
Andi Kleen11a8e772006-01-11 22:46:51 +01001262 apic_write(APIC_TASKPRI, value);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263
1264 /*
Vivek Goyalda7ed9f2006-03-25 16:31:16 +01001265 * After a crash, we no longer service the interrupts and a pending
1266 * interrupt from previous kernel might still have ISR bit set.
1267 *
1268 * Most probably by now CPU has serviced that pending interrupt and
1269 * it might not have done the ack_APIC_irq() because it thought,
1270 * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
1271 * does not clear the ISR bit and cpu thinks it has already serivced
1272 * the interrupt. Hence a vector might get locked. It was noticed
1273 * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
1274 */
Kerstin Jonsson8c3ba8d2010-05-24 12:13:15 -07001275 do {
1276 queued = 0;
1277 for (i = APIC_ISR_NR - 1; i >= 0; i--)
1278 queued |= apic_read(APIC_IRR + i*0x10);
1279
1280 for (i = APIC_ISR_NR - 1; i >= 0; i--) {
1281 value = apic_read(APIC_ISR + i*0x10);
1282 for (j = 31; j >= 0; j--) {
1283 if (value & (1<<j)) {
1284 ack_APIC_irq();
1285 acked++;
1286 }
1287 }
Vivek Goyalda7ed9f2006-03-25 16:31:16 +01001288 }
Kerstin Jonsson8c3ba8d2010-05-24 12:13:15 -07001289 if (acked > 256) {
1290 printk(KERN_ERR "LAPIC pending interrupts after %d EOI\n",
1291 acked);
1292 break;
1293 }
Shai Fultheim42fa4252012-04-20 01:12:32 +03001294 if (queued) {
Andy Lutomirskib47dcbd2014-10-15 10:12:07 -07001295 if (cpu_has_tsc && cpu_khz) {
Shai Fultheim42fa4252012-04-20 01:12:32 +03001296 rdtscll(ntsc);
1297 max_loops = (cpu_khz << 10) - (ntsc - tsc);
1298 } else
1299 max_loops--;
1300 }
Kerstin Jonsson8c3ba8d2010-05-24 12:13:15 -07001301 } while (queued && max_loops > 0);
1302 WARN_ON(max_loops <= 0);
Vivek Goyalda7ed9f2006-03-25 16:31:16 +01001303
1304 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305 * Now that we are all set up, enable the APIC
1306 */
1307 value = apic_read(APIC_SPIV);
1308 value &= ~APIC_VECTOR_MASK;
1309 /*
1310 * Enable APIC
1311 */
1312 value |= APIC_SPIV_APIC_ENABLED;
1313
Cyrill Gorcunov89c38c22008-08-24 02:01:43 -07001314#ifdef CONFIG_X86_32
1315 /*
1316 * Some unknown Intel IO/APIC (or APIC) errata is biting us with
1317 * certain networking cards. If high frequency interrupts are
1318 * happening on a particular IOAPIC pin, plus the IOAPIC routing
1319 * entry is masked/unmasked at a high rate as well then sooner or
1320 * later IOAPIC line gets 'stuck', no more interrupts are received
1321 * from the device. If focus CPU is disabled then the hang goes
1322 * away, oh well :-(
1323 *
1324 * [ This bug can be reproduced easily with a level-triggered
1325 * PCI Ne2000 networking cards and PII/PIII processors, dual
1326 * BX chipset. ]
1327 */
1328 /*
1329 * Actually disabling the focus CPU check just makes the hang less
1330 * frequent as it makes the interrupt distributon model be more
1331 * like LRU than MRU (the short-term load is more even across CPUs).
1332 * See also the comment in end_level_ioapic_irq(). --macro
1333 */
1334
1335 /*
1336 * - enable focus processor (bit==0)
1337 * - 64bit mode always use processor focus
1338 * so no need to set it
1339 */
1340 value &= ~APIC_SPIV_FOCUS_DISABLED;
1341#endif
Andi Kleen3f14c742006-09-26 10:52:29 +02001342
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343 /*
1344 * Set spurious IRQ vector
1345 */
1346 value |= SPURIOUS_APIC_VECTOR;
Andi Kleen11a8e772006-01-11 22:46:51 +01001347 apic_write(APIC_SPIV, value);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348
1349 /*
1350 * Set up LVT0, LVT1:
1351 *
1352 * set up through-local-APIC on the BP's LINT0. This is not
1353 * strictly necessary in pure symmetric-IO mode, but sometimes
1354 * we delegate interrupts to the 8259A.
1355 */
1356 /*
1357 * TODO: set up through-local-APIC from through-I/O-APIC? --macro
1358 */
1359 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
Tejun Heo0aa002f2010-12-09 11:47:21 +01001360 if (!cpu && (pic_mode || !value)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361 value = APIC_DM_EXTINT;
Tejun Heo0aa002f2010-12-09 11:47:21 +01001362 apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363 } else {
1364 value = APIC_DM_EXTINT | APIC_LVT_MASKED;
Tejun Heo0aa002f2010-12-09 11:47:21 +01001365 apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366 }
Andi Kleen11a8e772006-01-11 22:46:51 +01001367 apic_write(APIC_LVT0, value);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001368
1369 /*
1370 * only the BP should see the LINT1 NMI signal, obviously.
1371 */
Tejun Heo0aa002f2010-12-09 11:47:21 +01001372 if (!cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373 value = APIC_DM_NMI;
1374 else
1375 value = APIC_DM_NMI | APIC_LVT_MASKED;
Cyrill Gorcunov89c38c22008-08-24 02:01:43 -07001376 if (!lapic_is_integrated()) /* 82489DX */
1377 value |= APIC_LVT_LEVEL_TRIGGER;
Andi Kleen11a8e772006-01-11 22:46:51 +01001378 apic_write(APIC_LVT1, value);
Cyrill Gorcunov89c38c22008-08-24 02:01:43 -07001379
Andi Kleenbe71b852009-02-12 13:49:38 +01001380#ifdef CONFIG_X86_MCE_INTEL
1381 /* Recheck CMCI information after local APIC is up on CPU #0 */
Tejun Heo0aa002f2010-12-09 11:47:21 +01001382 if (!cpu)
Andi Kleenbe71b852009-02-12 13:49:38 +01001383 cmci_recheck();
1384#endif
Andi Kleen739f33b2008-01-30 13:30:40 +01001385}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386
Thomas Gleixner05f7e462015-01-15 21:22:40 +00001387static void end_local_APIC_setup(void)
Andi Kleen739f33b2008-01-30 13:30:40 +01001388{
1389 lapic_setup_esr();
Cyrill Gorcunovfa6b95f2008-08-18 20:45:58 +04001390
1391#ifdef CONFIG_X86_32
Cyrill Gorcunov1b4ee4e2008-08-18 23:12:33 +04001392 {
1393 unsigned int value;
1394 /* Disable the local apic timer */
1395 value = apic_read(APIC_LVTT);
1396 value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1397 apic_write(APIC_LVTT, value);
1398 }
Cyrill Gorcunovfa6b95f2008-08-18 20:45:58 +04001399#endif
1400
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401 apic_pm_activate();
Jan Beulich2fb270f2011-02-09 08:21:02 +00001402}
1403
Thomas Gleixner05f7e462015-01-15 21:22:40 +00001404/*
1405 * APIC setup function for application processors. Called from smpboot.c
1406 */
1407void apic_ap_setup(void)
Jan Beulich2fb270f2011-02-09 08:21:02 +00001408{
Thomas Gleixner05f7e462015-01-15 21:22:40 +00001409 setup_local_APIC();
Jan Beulich2fb270f2011-02-09 08:21:02 +00001410 end_local_APIC_setup();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001411}
1412
Yinghai Lu06cd9a72009-02-16 17:29:58 -08001413#ifdef CONFIG_X86_X2APIC
Thomas Gleixnerbfb05072015-01-15 21:22:12 +00001414int x2apic_mode;
Thomas Gleixner12e189d2015-01-15 21:22:22 +00001415
1416enum {
1417 X2APIC_OFF,
1418 X2APIC_ON,
1419 X2APIC_DISABLED,
1420};
1421static int x2apic_state;
1422
Thomas Gleixner44e25ff2015-01-15 21:22:24 +00001423static inline void __x2apic_disable(void)
1424{
1425 u64 msr;
1426
Thomas Gleixner659006b2015-01-15 21:22:26 +00001427 if (cpu_has_apic)
1428 return;
1429
Thomas Gleixner44e25ff2015-01-15 21:22:24 +00001430 rdmsrl(MSR_IA32_APICBASE, msr);
1431 if (!(msr & X2APIC_ENABLE))
1432 return;
1433 /* Disable xapic and x2apic first and then reenable xapic mode */
1434 wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
1435 wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
1436 printk_once(KERN_INFO "x2apic disabled\n");
1437}
1438
Thomas Gleixner659006b2015-01-15 21:22:26 +00001439static inline void __x2apic_enable(void)
1440{
1441 u64 msr;
1442
1443 rdmsrl(MSR_IA32_APICBASE, msr);
1444 if (msr & X2APIC_ENABLE)
1445 return;
1446 wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
1447 printk_once(KERN_INFO "x2apic enabled\n");
1448}
1449
Thomas Gleixnerbfb05072015-01-15 21:22:12 +00001450static int __init setup_nox2apic(char *str)
1451{
1452 if (x2apic_enabled()) {
1453 int apicid = native_apic_msr_read(APIC_ID);
1454
1455 if (apicid >= 255) {
1456 pr_warning("Apicid: %08x, cannot enforce nox2apic\n",
1457 apicid);
1458 return 0;
1459 }
Thomas Gleixner44e25ff2015-01-15 21:22:24 +00001460 pr_warning("x2apic already enabled.\n");
1461 __x2apic_disable();
1462 }
1463 setup_clear_cpu_cap(X86_FEATURE_X2APIC);
Thomas Gleixner12e189d2015-01-15 21:22:22 +00001464 x2apic_state = X2APIC_DISABLED;
Thomas Gleixner44e25ff2015-01-15 21:22:24 +00001465 x2apic_mode = 0;
Thomas Gleixnerbfb05072015-01-15 21:22:12 +00001466 return 0;
1467}
1468early_param("nox2apic", setup_nox2apic);
1469
Thomas Gleixner659006b2015-01-15 21:22:26 +00001470/* Called from cpu_init() to enable x2apic on (secondary) cpus */
1471void x2apic_setup(void)
1472{
1473 /*
1474 * If x2apic is not in ON state, disable it if already enabled
1475 * from BIOS.
1476 */
1477 if (x2apic_state != X2APIC_ON) {
1478 __x2apic_disable();
1479 return;
1480 }
1481 __x2apic_enable();
1482}
1483
Thomas Gleixner44e25ff2015-01-15 21:22:24 +00001484static __init void x2apic_disable(void)
Yinghai Lufb209bd2011-12-21 17:45:17 -08001485{
Thomas Gleixner6d2d49d2015-01-15 21:22:27 +00001486 u32 x2apic_id;
Yinghai Lufb209bd2011-12-21 17:45:17 -08001487
Thomas Gleixner6d2d49d2015-01-15 21:22:27 +00001488 if (x2apic_state != X2APIC_ON)
1489 goto out;
Yinghai Lufb209bd2011-12-21 17:45:17 -08001490
Thomas Gleixner6d2d49d2015-01-15 21:22:27 +00001491 x2apic_id = read_apic_id();
1492 if (x2apic_id >= 255)
1493 panic("Cannot disable x2apic, id: %08x\n", x2apic_id);
Yinghai Lufb209bd2011-12-21 17:45:17 -08001494
Thomas Gleixner6d2d49d2015-01-15 21:22:27 +00001495 __x2apic_disable();
1496 register_lapic_address(mp_lapic_addr);
1497out:
Thomas Gleixner12e189d2015-01-15 21:22:22 +00001498 x2apic_state = X2APIC_DISABLED;
Thomas Gleixner6d2d49d2015-01-15 21:22:27 +00001499 x2apic_mode = 0;
Yinghai Lufb209bd2011-12-21 17:45:17 -08001500}
1501
Thomas Gleixner659006b2015-01-15 21:22:26 +00001502static __init void x2apic_enable(void)
Suresh Siddha6e1cb382008-07-10 11:16:58 -07001503{
Thomas Gleixner659006b2015-01-15 21:22:26 +00001504 if (x2apic_state != X2APIC_OFF)
Yinghai Lu06cd9a72009-02-16 17:29:58 -08001505 return;
1506
Thomas Gleixner659006b2015-01-15 21:22:26 +00001507 x2apic_mode = 1;
Thomas Gleixner12e189d2015-01-15 21:22:22 +00001508 x2apic_state = X2APIC_ON;
Thomas Gleixner659006b2015-01-15 21:22:26 +00001509 __x2apic_enable();
Suresh Siddha6e1cb382008-07-10 11:16:58 -07001510}
Thomas Gleixnerd5241652015-01-15 21:22:17 +00001511
Thomas Gleixner62e61632015-01-15 21:22:21 +00001512static __init void try_to_enable_x2apic(int remap_mode)
Jiang Liu07806c52015-01-07 15:31:34 +08001513{
Thomas Gleixner659006b2015-01-15 21:22:26 +00001514 if (x2apic_state == X2APIC_DISABLED)
Jiang Liu07806c52015-01-07 15:31:34 +08001515 return;
1516
Thomas Gleixner62e61632015-01-15 21:22:21 +00001517 if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
Jiang Liu07806c52015-01-07 15:31:34 +08001518 /* IR is required if there is APIC ID > 255 even when running
1519 * under KVM
1520 */
1521 if (max_physical_apicid > 255 ||
Linus Torvalds8329aa92015-02-13 10:26:18 -08001522 !hypervisor_x2apic_available()) {
Thomas Gleixner62e61632015-01-15 21:22:21 +00001523 pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
Thomas Gleixner44e25ff2015-01-15 21:22:24 +00001524 x2apic_disable();
Jiang Liu07806c52015-01-07 15:31:34 +08001525 return;
1526 }
1527
1528 /*
1529 * without IR all CPUs can be addressed by IOAPIC/MSI
1530 * only in physical mode
1531 */
Thomas Gleixner55eae7d2015-01-15 21:22:19 +00001532 x2apic_phys = 1;
Jiang Liu07806c52015-01-07 15:31:34 +08001533 }
Thomas Gleixner659006b2015-01-15 21:22:26 +00001534 x2apic_enable();
Thomas Gleixner55eae7d2015-01-15 21:22:19 +00001535}
1536
1537void __init check_x2apic(void)
1538{
1539 if (x2apic_enabled()) {
1540 pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
1541 x2apic_mode = 1;
Thomas Gleixner12e189d2015-01-15 21:22:22 +00001542 x2apic_state = X2APIC_ON;
1543 } else if (!cpu_has_x2apic) {
1544 x2apic_state = X2APIC_DISABLED;
Thomas Gleixner55eae7d2015-01-15 21:22:19 +00001545 }
1546}
1547#else /* CONFIG_X86_X2APIC */
1548static int __init validate_x2apic(void)
1549{
1550 if (!apic_is_x2apic_enabled())
1551 return 0;
1552 /*
1553 * Checkme: Can we simply turn off x2apic here instead of panic?
1554 */
1555 panic("BIOS has enabled x2apic but kernel doesn't support x2apic, please disable x2apic in BIOS.\n");
1556}
1557early_initcall(validate_x2apic);
1558
Thomas Gleixner62e61632015-01-15 21:22:21 +00001559static inline void try_to_enable_x2apic(int remap_mode) { }
Thomas Gleixner659006b2015-01-15 21:22:26 +00001560static inline void __x2apic_enable(void) { }
Thomas Gleixner55eae7d2015-01-15 21:22:19 +00001561#endif /* !CONFIG_X86_X2APIC */
1562
1563static int __init try_to_enable_IR(void)
1564{
1565#ifdef CONFIG_X86_IO_APIC
1566 if (!x2apic_enabled() && skip_ioapic_setup) {
1567 pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
1568 return -1;
1569 }
Jiang Liu07806c52015-01-07 15:31:34 +08001570#endif
Thomas Gleixner55eae7d2015-01-15 21:22:19 +00001571 return irq_remapping_enable();
Gleb Natapovce69a782009-07-20 15:24:17 +03001572}
1573
1574void __init enable_IR_x2apic(void)
1575{
1576 unsigned long flags;
Jiang Liu07806c52015-01-07 15:31:34 +08001577 int ret, ir_stat;
Yinghai Lub7f42ab2009-08-17 11:19:40 -07001578
Jiang Liu07806c52015-01-07 15:31:34 +08001579 ir_stat = irq_remapping_prepare();
1580 if (ir_stat < 0 && !x2apic_supported())
Yinghai Lue6707612009-11-21 00:23:37 -08001581 return;
Gleb Natapovce69a782009-07-20 15:24:17 +03001582
Suresh Siddha31dce142011-05-18 16:31:33 -07001583 ret = save_ioapic_entries();
Cyrill Gorcunov5ffa4eb2008-09-18 23:37:57 +04001584 if (ret) {
Cyrill Gorcunovba21ebb2008-11-10 09:16:41 +01001585 pr_info("Saving IO-APIC state failed: %d\n", ret);
Yinghai Lufb209bd2011-12-21 17:45:17 -08001586 return;
Cyrill Gorcunov5ffa4eb2008-09-18 23:37:57 +04001587 }
Suresh Siddha6e1cb382008-07-10 11:16:58 -07001588
Suresh Siddha05c3dc22009-03-16 17:05:03 -07001589 local_irq_save(flags);
Jacob Panb81bb372009-11-09 11:27:04 -08001590 legacy_pic->mask_all();
Suresh Siddha31dce142011-05-18 16:31:33 -07001591 mask_ioapic_entries();
Suresh Siddha05c3dc22009-03-16 17:05:03 -07001592
Jiang Liu07806c52015-01-07 15:31:34 +08001593 /* If irq_remapping_prepare() succeded, try to enable it */
1594 if (ir_stat >= 0)
1595 ir_stat = try_to_enable_IR();
1596 /* ir_stat contains the remap mode or an error code */
1597 try_to_enable_x2apic(ir_stat);
Yinghai Lua31bc322011-12-23 11:01:43 -08001598
Jiang Liu07806c52015-01-07 15:31:34 +08001599 if (ir_stat < 0)
Suresh Siddha31dce142011-05-18 16:31:33 -07001600 restore_ioapic_entries();
Jacob Panb81bb372009-11-09 11:27:04 -08001601 legacy_pic->restore_mask();
Suresh Siddha6e1cb382008-07-10 11:16:58 -07001602 local_irq_restore(flags);
Suresh Siddha6e1cb382008-07-10 11:16:58 -07001603}
Weidong Han93758232009-04-17 16:42:14 +08001604
Yinghai Lube7a6562008-08-24 02:01:51 -07001605#ifdef CONFIG_X86_64
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001606/*
1607 * Detect and enable local APICs on non-SMP boards.
1608 * Original code written by Keir Fraser.
1609 * On AMD64 we trust the BIOS - if it says no APIC it is likely
1610 * not correctly set up (usually the APIC timer won't work etc.)
1611 */
1612static int __init detect_init_APIC(void)
1613{
1614 if (!cpu_has_apic) {
Cyrill Gorcunovba21ebb2008-11-10 09:16:41 +01001615 pr_info("No local APIC present\n");
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001616 return -1;
1617 }
1618
1619 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001620 return 0;
1621}
Yinghai Lube7a6562008-08-24 02:01:51 -07001622#else
Thomas Gleixner5a7ae782010-10-19 10:46:28 -07001623
Henrik Kretzschmar25874a22011-03-11 08:02:36 +01001624static int __init apic_verify(void)
Thomas Gleixner5a7ae782010-10-19 10:46:28 -07001625{
1626 u32 features, h, l;
1627
1628 /*
1629 * The APIC feature bit should now be enabled
1630 * in `cpuid'
1631 */
1632 features = cpuid_edx(1);
1633 if (!(features & (1 << X86_FEATURE_APIC))) {
1634 pr_warning("Could not enable APIC!\n");
1635 return -1;
1636 }
1637 set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1638 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
1639
1640 /* The BIOS may have set up the APIC at some other address */
Bryan O'Donoghuecbf28292012-04-18 17:37:39 +01001641 if (boot_cpu_data.x86 >= 6) {
1642 rdmsr(MSR_IA32_APICBASE, l, h);
1643 if (l & MSR_IA32_APICBASE_ENABLE)
1644 mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
1645 }
Thomas Gleixner5a7ae782010-10-19 10:46:28 -07001646
1647 pr_info("Found and enabled local APIC!\n");
1648 return 0;
1649}
1650
Henrik Kretzschmar25874a22011-03-11 08:02:36 +01001651int __init apic_force_enable(unsigned long addr)
Thomas Gleixner5a7ae782010-10-19 10:46:28 -07001652{
1653 u32 h, l;
1654
1655 if (disable_apic)
1656 return -1;
1657
1658 /*
1659 * Some BIOSes disable the local APIC in the APIC_BASE
1660 * MSR. This can only be done in software for Intel P6 or later
1661 * and AMD K7 (Model > 1) or later.
1662 */
Bryan O'Donoghuecbf28292012-04-18 17:37:39 +01001663 if (boot_cpu_data.x86 >= 6) {
1664 rdmsr(MSR_IA32_APICBASE, l, h);
1665 if (!(l & MSR_IA32_APICBASE_ENABLE)) {
1666 pr_info("Local APIC disabled by BIOS -- reenabling.\n");
1667 l &= ~MSR_IA32_APICBASE_BASE;
1668 l |= MSR_IA32_APICBASE_ENABLE | addr;
1669 wrmsr(MSR_IA32_APICBASE, l, h);
1670 enabled_via_apicbase = 1;
1671 }
Thomas Gleixner5a7ae782010-10-19 10:46:28 -07001672 }
1673 return apic_verify();
1674}
1675
Yinghai Lube7a6562008-08-24 02:01:51 -07001676/*
1677 * Detect and initialize APIC
1678 */
1679static int __init detect_init_APIC(void)
1680{
Yinghai Lube7a6562008-08-24 02:01:51 -07001681 /* Disabled by kernel option? */
1682 if (disable_apic)
1683 return -1;
1684
1685 switch (boot_cpu_data.x86_vendor) {
1686 case X86_VENDOR_AMD:
1687 if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
Borislav Petkov85877062009-02-03 16:24:22 +01001688 (boot_cpu_data.x86 >= 15))
Yinghai Lube7a6562008-08-24 02:01:51 -07001689 break;
1690 goto no_apic;
1691 case X86_VENDOR_INTEL:
1692 if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
1693 (boot_cpu_data.x86 == 5 && cpu_has_apic))
1694 break;
1695 goto no_apic;
1696 default:
1697 goto no_apic;
1698 }
1699
1700 if (!cpu_has_apic) {
1701 /*
1702 * Over-ride BIOS and try to enable the local APIC only if
1703 * "lapic" specified.
1704 */
1705 if (!force_enable_local_apic) {
Cyrill Gorcunovba21ebb2008-11-10 09:16:41 +01001706 pr_info("Local APIC disabled by BIOS -- "
1707 "you can enable it with \"lapic\"\n");
Yinghai Lube7a6562008-08-24 02:01:51 -07001708 return -1;
1709 }
Thomas Gleixnera906fda2011-02-25 16:09:31 +01001710 if (apic_force_enable(APIC_DEFAULT_PHYS_BASE))
Thomas Gleixner5a7ae782010-10-19 10:46:28 -07001711 return -1;
1712 } else {
1713 if (apic_verify())
1714 return -1;
Yinghai Lube7a6562008-08-24 02:01:51 -07001715 }
Yinghai Lube7a6562008-08-24 02:01:51 -07001716
1717 apic_pm_activate();
1718
1719 return 0;
1720
1721no_apic:
Cyrill Gorcunovba21ebb2008-11-10 09:16:41 +01001722 pr_info("No local APIC present or hardware disabled\n");
Yinghai Lube7a6562008-08-24 02:01:51 -07001723 return -1;
1724}
1725#endif
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001726
1727/**
1728 * init_apic_mappings - initialize APIC mappings
1729 */
1730void __init init_apic_mappings(void)
1731{
Yinghai Lu4401da62009-05-02 10:40:57 -07001732 unsigned int new_apicid;
1733
Suresh Siddhafc1edaf2009-04-20 13:02:27 -07001734 if (x2apic_mode) {
Yinghai Lu4c9961d2008-07-11 18:44:16 -07001735 boot_cpu_physical_apicid = read_apic_id();
Suresh Siddha6e1cb382008-07-10 11:16:58 -07001736 return;
1737 }
1738
Yinghai Lu4797f6b2009-05-02 10:40:57 -07001739 /* If no local APIC can be found return early */
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001740 if (!smp_found_config && detect_init_APIC()) {
Yinghai Lu4797f6b2009-05-02 10:40:57 -07001741 /* lets NOP'ify apic operations */
Cyrill Gorcunovcec6be62009-05-11 17:41:40 +04001742 pr_info("APIC: disable apic facility\n");
1743 apic_disable();
Yinghai Lu4797f6b2009-05-02 10:40:57 -07001744 } else {
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001745 apic_phys = mp_lapic_addr;
1746
Yinghai Lu4797f6b2009-05-02 10:40:57 -07001747 /*
1748 * acpi lapic path already maps that address in
1749 * acpi_register_lapic_address()
1750 */
Eric W. Biederman5989cd62010-08-04 13:30:27 -07001751 if (!acpi_lapic && !smp_found_config)
Yinghai Lu326a2e62010-12-07 00:55:38 -08001752 register_lapic_address(apic_phys);
Cyrill Gorcunovcec6be62009-05-11 17:41:40 +04001753 }
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001754
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001755 /*
1756 * Fetch the APIC ID of the BSP in case we have a
1757 * default configuration (or the MP table is broken).
1758 */
Yinghai Lu4401da62009-05-02 10:40:57 -07001759 new_apicid = read_apic_id();
1760 if (boot_cpu_physical_apicid != new_apicid) {
1761 boot_cpu_physical_apicid = new_apicid;
Cyrill Gorcunov103428e2009-06-07 16:48:40 +04001762 /*
1763 * yeah -- we lie about apic_version
1764 * in case if apic was disabled via boot option
1765 * but it's not a problem for SMP compiled kernel
1766 * since smp_sanity_check is prepared for such a case
1767 * and disable smp mode
1768 */
Yinghai Lu4401da62009-05-02 10:40:57 -07001769 apic_version[new_apicid] =
1770 GET_APIC_VERSION(apic_read(APIC_LVR));
Cyrill Gorcunov08306ce2009-04-12 20:47:41 +04001771 }
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001772}
1773
Yinghai Luc0104d32010-12-07 00:55:17 -08001774void __init register_lapic_address(unsigned long address)
1775{
1776 mp_lapic_addr = address;
1777
Yinghai Lu04501932010-12-07 00:55:56 -08001778 if (!x2apic_mode) {
1779 set_fixmap_nocache(FIX_APIC_BASE, address);
1780 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
1781 APIC_BASE, mp_lapic_addr);
1782 }
Yinghai Luc0104d32010-12-07 00:55:17 -08001783 if (boot_cpu_physical_apicid == -1U) {
1784 boot_cpu_physical_apicid = read_apic_id();
1785 apic_version[boot_cpu_physical_apicid] =
1786 GET_APIC_VERSION(apic_read(APIC_LVR));
1787 }
1788}
1789
Yinghai Lu56d91f12010-12-16 19:09:24 -08001790int apic_version[MAX_LOCAL_APIC];
Cyrill Gorcunov1b313f42008-08-18 20:45:57 +04001791
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001792/*
1793 * Local APIC interrupts
1794 */
1795
1796/*
1797 * This interrupt should _never_ happen with our APIC/SMP architecture
1798 */
Jan Beulich2414e022014-11-03 08:39:43 +00001799static inline void __smp_spurious_interrupt(u8 vector)
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001800{
Yinghai Ludc1528d2008-08-24 02:01:53 -07001801 u32 v;
1802
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001803 /*
1804 * Check if this really is a spurious interrupt and ACK it
1805 * if it is a vectored one. Just in case...
1806 * Spurious interrupts should not be ACKed.
1807 */
Jan Beulich2414e022014-11-03 08:39:43 +00001808 v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
1809 if (v & (1 << (vector & 0x1f)))
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001810 ack_APIC_irq();
1811
Hiroshi Shimamoto915b0d02008-12-08 19:19:26 -08001812 inc_irq_stat(irq_spurious_count);
1813
Yinghai Ludc1528d2008-08-24 02:01:53 -07001814 /* see sw-dev-man vol 3, chapter 7.4.13.5 */
Jan Beulich2414e022014-11-03 08:39:43 +00001815 pr_info("spurious APIC interrupt through vector %02x on CPU#%d, "
1816 "should never happen.\n", vector, smp_processor_id());
Seiji Aguchieddc0e92013-06-20 11:45:17 -04001817}
1818
Andi Kleen1d9090e2013-08-05 15:02:37 -07001819__visible void smp_spurious_interrupt(struct pt_regs *regs)
Seiji Aguchieddc0e92013-06-20 11:45:17 -04001820{
1821 entering_irq();
Jan Beulich2414e022014-11-03 08:39:43 +00001822 __smp_spurious_interrupt(~regs->orig_ax);
Seiji Aguchieddc0e92013-06-20 11:45:17 -04001823 exiting_irq();
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001824}
1825
Andi Kleen1d9090e2013-08-05 15:02:37 -07001826__visible void smp_trace_spurious_interrupt(struct pt_regs *regs)
Seiji Aguchicf910e82013-06-20 11:46:53 -04001827{
Jan Beulich2414e022014-11-03 08:39:43 +00001828 u8 vector = ~regs->orig_ax;
1829
Seiji Aguchicf910e82013-06-20 11:46:53 -04001830 entering_irq();
Jan Beulich2414e022014-11-03 08:39:43 +00001831 trace_spurious_apic_entry(vector);
1832 __smp_spurious_interrupt(vector);
1833 trace_spurious_apic_exit(vector);
Seiji Aguchicf910e82013-06-20 11:46:53 -04001834 exiting_irq();
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001835}
1836
1837/*
1838 * This interrupt should never happen with our APIC/SMP architecture
1839 */
Seiji Aguchieddc0e92013-06-20 11:45:17 -04001840static inline void __smp_error_interrupt(struct pt_regs *regs)
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001841{
Richard Weinberger60283df2014-01-14 08:44:47 +01001842 u32 v;
Youquan Song2b398bd2011-04-14 14:36:08 +08001843 u32 i = 0;
1844 static const char * const error_interrupt_reason[] = {
1845 "Send CS error", /* APIC Error Bit 0 */
1846 "Receive CS error", /* APIC Error Bit 1 */
1847 "Send accept error", /* APIC Error Bit 2 */
1848 "Receive accept error", /* APIC Error Bit 3 */
1849 "Redirectable IPI", /* APIC Error Bit 4 */
1850 "Send illegal vector", /* APIC Error Bit 5 */
1851 "Received illegal vector", /* APIC Error Bit 6 */
1852 "Illegal register address", /* APIC Error Bit 7 */
1853 };
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001854
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001855 /* First tickle the hardware, only then report what went on. -- REW */
Maciej W. Rozycki023de4a2014-04-01 13:30:21 +01001856 if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
1857 apic_write(APIC_ESR, 0);
Richard Weinberger60283df2014-01-14 08:44:47 +01001858 v = apic_read(APIC_ESR);
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001859 ack_APIC_irq();
1860 atomic_inc(&irq_err_count);
1861
Richard Weinberger60283df2014-01-14 08:44:47 +01001862 apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x",
1863 smp_processor_id(), v);
Youquan Song2b398bd2011-04-14 14:36:08 +08001864
Richard Weinberger60283df2014-01-14 08:44:47 +01001865 v &= 0xff;
1866 while (v) {
1867 if (v & 0x1)
Youquan Song2b398bd2011-04-14 14:36:08 +08001868 apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
1869 i++;
Richard Weinberger60283df2014-01-14 08:44:47 +01001870 v >>= 1;
Peter Senna Tschudin4b8073e2012-09-18 18:36:14 +02001871 }
Youquan Song2b398bd2011-04-14 14:36:08 +08001872
1873 apic_printk(APIC_DEBUG, KERN_CONT "\n");
1874
Seiji Aguchieddc0e92013-06-20 11:45:17 -04001875}
1876
Andi Kleen1d9090e2013-08-05 15:02:37 -07001877__visible void smp_error_interrupt(struct pt_regs *regs)
Seiji Aguchieddc0e92013-06-20 11:45:17 -04001878{
1879 entering_irq();
1880 __smp_error_interrupt(regs);
1881 exiting_irq();
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001882}
1883
Andi Kleen1d9090e2013-08-05 15:02:37 -07001884__visible void smp_trace_error_interrupt(struct pt_regs *regs)
Seiji Aguchicf910e82013-06-20 11:46:53 -04001885{
1886 entering_irq();
1887 trace_error_apic_entry(ERROR_APIC_VECTOR);
1888 __smp_error_interrupt(regs);
1889 trace_error_apic_exit(ERROR_APIC_VECTOR);
1890 exiting_irq();
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001891}
1892
Glauber Costab5841762008-05-28 13:38:28 -03001893/**
Cyrill Gorcunov36c9d672008-08-18 20:45:53 +04001894 * connect_bsp_APIC - attach the APIC to the interrupt system
1895 */
Thomas Gleixner05f7e462015-01-15 21:22:40 +00001896static void __init connect_bsp_APIC(void)
Glauber Costab5841762008-05-28 13:38:28 -03001897{
Cyrill Gorcunov36c9d672008-08-18 20:45:53 +04001898#ifdef CONFIG_X86_32
1899 if (pic_mode) {
1900 /*
1901 * Do not trust the local APIC being empty at bootup.
1902 */
1903 clear_local_APIC();
1904 /*
1905 * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
1906 * local APIC to INT and NMI lines.
1907 */
1908 apic_printk(APIC_VERBOSE, "leaving PIC mode, "
1909 "enabling APIC mode.\n");
Cyrill Gorcunovc0eaa452009-04-12 20:47:40 +04001910 imcr_pic_to_apic();
Cyrill Gorcunov36c9d672008-08-18 20:45:53 +04001911 }
1912#endif
Glauber Costab5841762008-05-28 13:38:28 -03001913}
1914
Cyrill Gorcunov274cfe52008-08-16 23:21:53 +04001915/**
1916 * disconnect_bsp_APIC - detach the APIC from the interrupt system
1917 * @virt_wire_setup: indicates, whether virtual wire mode is selected
1918 *
1919 * Virtual wire mode is necessary to deliver legacy interrupts even when the
1920 * APIC is disabled.
1921 */
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001922void disconnect_bsp_APIC(int virt_wire_setup)
1923{
Cyrill Gorcunov1b4ee4e2008-08-18 23:12:33 +04001924 unsigned int value;
1925
Cyrill Gorcunovc177b0b2008-08-18 20:45:56 +04001926#ifdef CONFIG_X86_32
1927 if (pic_mode) {
1928 /*
1929 * Put the board back into PIC mode (has an effect only on
1930 * certain older boards). Note that APIC interrupts, including
1931 * IPIs, won't work beyond this point! The only exception are
1932 * INIT IPIs.
1933 */
1934 apic_printk(APIC_VERBOSE, "disabling APIC mode, "
1935 "entering PIC mode.\n");
Cyrill Gorcunovc0eaa452009-04-12 20:47:40 +04001936 imcr_apic_to_pic();
Cyrill Gorcunovc177b0b2008-08-18 20:45:56 +04001937 return;
1938 }
1939#endif
1940
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001941 /* Go back to Virtual Wire compatibility mode */
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001942
1943 /* For the spurious interrupt use vector F, and enable it */
1944 value = apic_read(APIC_SPIV);
1945 value &= ~APIC_VECTOR_MASK;
1946 value |= APIC_SPIV_APIC_ENABLED;
1947 value |= 0xf;
1948 apic_write(APIC_SPIV, value);
1949
1950 if (!virt_wire_setup) {
1951 /*
1952 * For LVT0 make it edge triggered, active high,
1953 * external and enabled
1954 */
1955 value = apic_read(APIC_LVT0);
1956 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
1957 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
1958 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
1959 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
1960 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
1961 apic_write(APIC_LVT0, value);
1962 } else {
1963 /* Disable LVT0 */
1964 apic_write(APIC_LVT0, APIC_LVT_MASKED);
1965 }
1966
Cyrill Gorcunovc177b0b2008-08-18 20:45:56 +04001967 /*
1968 * For LVT1 make it edge triggered, active high,
1969 * nmi and enabled
1970 */
Thomas Gleixner0e078e22008-01-30 13:30:20 +01001971 value = apic_read(APIC_LVT1);
1972 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
1973 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
1974 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
1975 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
1976 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
1977 apic_write(APIC_LVT1, value);
1978}
1979
Jiang Liu7e1f85f2013-09-02 11:57:36 +08001980int generic_processor_info(int apicid, int version)
Alexey Starikovskiybe8a5682008-03-27 23:56:19 +03001981{
Vivek Goyal14cb6dc2011-07-08 13:19:26 -04001982 int cpu, max = nr_cpu_ids;
1983 bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid,
1984 phys_cpu_present_map);
1985
1986 /*
HATAYAMA Daisuke151e0c72014-01-15 15:44:58 +09001987 * boot_cpu_physical_apicid is designed to have the apicid
1988 * returned by read_apic_id(), i.e, the apicid of the
1989 * currently booting-up processor. However, on some platforms,
H. Peter Anvin5b4d1db2014-01-15 13:02:08 -08001990 * it is temporarily modified by the apicid reported as BSP
HATAYAMA Daisuke151e0c72014-01-15 15:44:58 +09001991 * through MP table. Concretely:
1992 *
1993 * - arch/x86/kernel/mpparse.c: MP_processor_info()
1994 * - arch/x86/mm/amdtopology.c: amd_numa_init()
HATAYAMA Daisuke151e0c72014-01-15 15:44:58 +09001995 *
1996 * This function is executed with the modified
1997 * boot_cpu_physical_apicid. So, disabled_cpu_apicid kernel
1998 * parameter doesn't work to disable APs on kdump 2nd kernel.
1999 *
2000 * Since fixing handling of boot_cpu_physical_apicid requires
2001 * another discussion and tests on each platform, we leave it
2002 * for now and here we use read_apic_id() directly in this
2003 * function, generic_processor_info().
2004 */
2005 if (disabled_cpu_apicid != BAD_APICID &&
2006 disabled_cpu_apicid != read_apic_id() &&
2007 disabled_cpu_apicid == apicid) {
2008 int thiscpu = num_processors + disabled_cpus;
2009
H. Peter Anvin5b4d1db2014-01-15 13:02:08 -08002010 pr_warning("APIC: Disabling requested cpu."
HATAYAMA Daisuke151e0c72014-01-15 15:44:58 +09002011 " Processor %d/0x%x ignored.\n",
2012 thiscpu, apicid);
2013
2014 disabled_cpus++;
2015 return -ENODEV;
2016 }
2017
2018 /*
Vivek Goyal14cb6dc2011-07-08 13:19:26 -04002019 * If boot cpu has not been detected yet, then only allow upto
2020 * nr_cpu_ids - 1 processors and keep one slot free for boot cpu
2021 */
2022 if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 &&
2023 apicid != boot_cpu_physical_apicid) {
2024 int thiscpu = max + disabled_cpus - 1;
2025
2026 pr_warning(
2027 "ACPI: NR_CPUS/possible_cpus limit of %i almost"
2028 " reached. Keeping one slot for boot cpu."
2029 " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2030
2031 disabled_cpus++;
Jiang Liu7e1f85f2013-09-02 11:57:36 +08002032 return -ENODEV;
Vivek Goyal14cb6dc2011-07-08 13:19:26 -04002033 }
Alexey Starikovskiybe8a5682008-03-27 23:56:19 +03002034
Mike Travis3b11ce72008-12-17 15:21:39 -08002035 if (num_processors >= nr_cpu_ids) {
Mike Travis3b11ce72008-12-17 15:21:39 -08002036 int thiscpu = max + disabled_cpus;
2037
2038 pr_warning(
2039 "ACPI: NR_CPUS/possible_cpus limit of %i reached."
2040 " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2041
2042 disabled_cpus++;
Jiang Liu7e1f85f2013-09-02 11:57:36 +08002043 return -EINVAL;
Alexey Starikovskiybe8a5682008-03-27 23:56:19 +03002044 }
2045
2046 num_processors++;
Alexey Starikovskiybe8a5682008-03-27 23:56:19 +03002047 if (apicid == boot_cpu_physical_apicid) {
2048 /*
2049 * x86_bios_cpu_apicid is required to have processors listed
2050 * in same order as logical cpu numbers. Hence the first
2051 * entry is BSP, and so on.
Yinghai Lue5fea862011-02-08 23:22:17 -08002052 * boot_cpu_init() already hold bit 0 in cpu_present_mask
2053 * for BSP.
Alexey Starikovskiybe8a5682008-03-27 23:56:19 +03002054 */
2055 cpu = 0;
Yinghai Lue5fea862011-02-08 23:22:17 -08002056 } else
2057 cpu = cpumask_next_zero(-1, cpu_present_mask);
2058
2059 /*
2060 * Validate version
2061 */
2062 if (version == 0x0) {
2063 pr_warning("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n",
2064 cpu, apicid);
2065 version = 0x10;
Alexey Starikovskiybe8a5682008-03-27 23:56:19 +03002066 }
Yinghai Lue5fea862011-02-08 23:22:17 -08002067 apic_version[apicid] = version;
2068
2069 if (version != apic_version[boot_cpu_physical_apicid]) {
2070 pr_warning("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
2071 apic_version[boot_cpu_physical_apicid], cpu, version);
2072 }
2073
2074 physid_set(apicid, phys_cpu_present_map);
Yinghai Lue0da3362008-06-08 18:29:22 -07002075 if (apicid > max_physical_apicid)
2076 max_physical_apicid = apicid;
2077
Ingo Molnar3e5095d2009-01-27 17:07:08 +01002078#if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
Tejun Heof10fcd42009-01-13 20:41:34 +09002079 early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
2080 early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
Cyrill Gorcunov1b313f42008-08-18 20:45:57 +04002081#endif
Tejun Heoacb8bc02011-01-23 14:37:33 +01002082#ifdef CONFIG_X86_32
2083 early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
2084 apic->x86_32_early_logical_apicid(cpu);
2085#endif
Mike Travis1de88cd2008-12-16 17:34:02 -08002086 set_cpu_possible(cpu, true);
2087 set_cpu_present(cpu, true);
Jiang Liu7e1f85f2013-09-02 11:57:36 +08002088
2089 return cpu;
Alexey Starikovskiybe8a5682008-03-27 23:56:19 +03002090}
2091
Suresh Siddha0c81c742008-07-10 11:16:48 -07002092int hard_smp_processor_id(void)
2093{
2094 return read_apic_id();
2095}
Ingo Molnar1dcdd3d2009-01-28 17:55:37 +01002096
2097void default_init_apic_ldr(void)
2098{
2099 unsigned long val;
2100
2101 apic_write(APIC_DFR, APIC_DFR_VALUE);
2102 val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
2103 val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
2104 apic_write(APIC_LDR, val);
2105}
2106
Alexander Gordeevff164322012-06-07 15:15:59 +02002107int default_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
2108 const struct cpumask *andmask,
2109 unsigned int *apicid)
Alexander Gordeev63982682012-06-05 13:23:44 +02002110{
Alexander Gordeevea3807e2012-06-14 09:49:55 +02002111 unsigned int cpu;
Alexander Gordeev63982682012-06-05 13:23:44 +02002112
2113 for_each_cpu_and(cpu, cpumask, andmask) {
2114 if (cpumask_test_cpu(cpu, cpu_online_mask))
2115 break;
2116 }
Alexander Gordeevff164322012-06-07 15:15:59 +02002117
Alexander Gordeevea3807e2012-06-14 09:49:55 +02002118 if (likely(cpu < nr_cpu_ids)) {
Alexander Gordeeva5a39152012-06-14 09:49:35 +02002119 *apicid = per_cpu(x86_cpu_to_apicid, cpu);
2120 return 0;
Alexander Gordeeva5a39152012-06-14 09:49:35 +02002121 }
Alexander Gordeevea3807e2012-06-14 09:49:55 +02002122
2123 return -EINVAL;
Alexander Gordeev63982682012-06-05 13:23:44 +02002124}
2125
Thomas Gleixner0e078e22008-01-30 13:30:20 +01002126/*
Michael S. Tsirkin1551df62012-07-15 15:56:46 +03002127 * Override the generic EOI implementation with an optimized version.
2128 * Only called during early boot when only one CPU is active and with
2129 * interrupts disabled, so we know this does not race with actual APIC driver
2130 * use.
2131 */
2132void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v))
2133{
2134 struct apic **drv;
2135
2136 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
2137 /* Should happen once for each apic */
2138 WARN_ON((*drv)->eoi_write == eoi_write);
2139 (*drv)->eoi_write = eoi_write;
2140 }
2141}
2142
Thomas Gleixner374aab32015-01-15 21:22:44 +00002143static void __init apic_bsp_up_setup(void)
Thomas Gleixner05f7e462015-01-15 21:22:40 +00002144{
Thomas Gleixner374aab32015-01-15 21:22:44 +00002145#ifdef CONFIG_X86_64
2146 apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid));
2147#else
Thomas Gleixner05f7e462015-01-15 21:22:40 +00002148 /*
Thomas Gleixner374aab32015-01-15 21:22:44 +00002149 * Hack: In case of kdump, after a crash, kernel might be booting
2150 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
2151 * might be zero if read from MP tables. Get it from LAPIC.
Thomas Gleixner05f7e462015-01-15 21:22:40 +00002152 */
Thomas Gleixner374aab32015-01-15 21:22:44 +00002153# ifdef CONFIG_CRASH_DUMP
2154 boot_cpu_physical_apicid = read_apic_id();
2155# endif
2156#endif
2157 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
Thomas Gleixner05f7e462015-01-15 21:22:40 +00002158}
2159
2160/**
2161 * apic_bsp_setup - Setup function for local apic and io-apic
Thomas Gleixner374aab32015-01-15 21:22:44 +00002162 * @upmode: Force UP mode (for APIC_init_uniprocessor)
Thomas Gleixner05f7e462015-01-15 21:22:40 +00002163 *
2164 * Returns:
2165 * apic_id of BSP APIC
2166 */
Thomas Gleixner374aab32015-01-15 21:22:44 +00002167int __init apic_bsp_setup(bool upmode)
Thomas Gleixner05f7e462015-01-15 21:22:40 +00002168{
2169 int id;
2170
2171 connect_bsp_APIC();
Thomas Gleixner374aab32015-01-15 21:22:44 +00002172 if (upmode)
2173 apic_bsp_up_setup();
Thomas Gleixner05f7e462015-01-15 21:22:40 +00002174 setup_local_APIC();
2175
2176 if (x2apic_mode)
2177 id = apic_read(APIC_LDR);
2178 else
2179 id = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
2180
2181 enable_IO_APIC();
Thomas Gleixner374aab32015-01-15 21:22:44 +00002182 end_local_APIC_setup();
2183 irq_remap_enable_fault_handling();
Thomas Gleixner05f7e462015-01-15 21:22:40 +00002184 setup_IO_APIC();
Thomas Gleixner9c4d9c72015-01-15 21:22:45 +00002185 /* Setup local timer */
2186 x86_init.timers.setup_percpu_clockev();
Thomas Gleixner05f7e462015-01-15 21:22:40 +00002187 return id;
2188}
2189
Michael S. Tsirkin1551df62012-07-15 15:56:46 +03002190/*
Thomas Gleixnere714a912015-01-15 21:22:37 +00002191 * This initializes the IO-APIC and APIC hardware if this is
2192 * a UP kernel.
2193 */
2194int __init APIC_init_uniprocessor(void)
2195{
2196 if (disable_apic) {
2197 pr_info("Apic disabled\n");
2198 return -1;
2199 }
2200#ifdef CONFIG_X86_64
2201 if (!cpu_has_apic) {
2202 disable_apic = 1;
2203 pr_info("Apic disabled by BIOS\n");
2204 return -1;
2205 }
2206#else
2207 if (!smp_found_config && !cpu_has_apic)
2208 return -1;
2209
2210 /*
2211 * Complain if the BIOS pretends there is one.
2212 */
2213 if (!cpu_has_apic &&
2214 APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
2215 pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
2216 boot_cpu_physical_apicid);
2217 return -1;
2218 }
2219#endif
2220
Thomas Gleixner374aab32015-01-15 21:22:44 +00002221 if (!smp_found_config)
2222 disable_ioapic_support();
2223
Thomas Gleixnere714a912015-01-15 21:22:37 +00002224 default_setup_apic_routing();
Thomas Gleixner374aab32015-01-15 21:22:44 +00002225 apic_bsp_setup(true);
Thomas Gleixnere714a912015-01-15 21:22:37 +00002226 return 0;
2227}
2228
Thomas Gleixner30b8b002015-01-15 21:22:39 +00002229#ifdef CONFIG_UP_LATE_INIT
2230void __init up_late_init(void)
2231{
2232 APIC_init_uniprocessor();
2233}
2234#endif
2235
Thomas Gleixnere714a912015-01-15 21:22:37 +00002236/*
Thomas Gleixner0e078e22008-01-30 13:30:20 +01002237 * Power management
2238 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002239#ifdef CONFIG_PM
2240
2241static struct {
Cyrill Gorcunov274cfe52008-08-16 23:21:53 +04002242 /*
2243 * 'active' is true if the local APIC was enabled by us and
2244 * not the BIOS; this signifies that we are also responsible
2245 * for disabling it before entering apm/acpi suspend
2246 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002247 int active;
2248 /* r/w apic fields */
2249 unsigned int apic_id;
2250 unsigned int apic_taskpri;
2251 unsigned int apic_ldr;
2252 unsigned int apic_dfr;
2253 unsigned int apic_spiv;
2254 unsigned int apic_lvtt;
2255 unsigned int apic_lvtpc;
2256 unsigned int apic_lvt0;
2257 unsigned int apic_lvt1;
2258 unsigned int apic_lvterr;
2259 unsigned int apic_tmict;
2260 unsigned int apic_tdcr;
2261 unsigned int apic_thmr;
2262} apic_pm_state;
2263
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01002264static int lapic_suspend(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002265{
2266 unsigned long flags;
Karsten Wiesef990fff2006-12-07 02:14:11 +01002267 int maxlvt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002268
2269 if (!apic_pm_state.active)
2270 return 0;
2271
Thomas Gleixner37e650c2008-01-30 13:30:14 +01002272 maxlvt = lapic_get_maxlvt();
Karsten Wiesef990fff2006-12-07 02:14:11 +01002273
Suresh Siddha2d7a66d2008-07-11 14:24:19 -07002274 apic_pm_state.apic_id = apic_read(APIC_ID);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002275 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2276 apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2277 apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2278 apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2279 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
Karsten Wiesef990fff2006-12-07 02:14:11 +01002280 if (maxlvt >= 4)
2281 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002282 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2283 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2284 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2285 apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2286 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
Andi Kleen4efc0672009-04-28 19:07:31 +02002287#ifdef CONFIG_X86_THERMAL_VECTOR
Karsten Wiesef990fff2006-12-07 02:14:11 +01002288 if (maxlvt >= 5)
2289 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2290#endif
Cyrill Gorcunov24968cf2008-08-16 23:21:52 +04002291
Fernando Luis Vázquez Cao2b94ab22006-09-26 10:52:33 +02002292 local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002293 disable_local_APIC();
Suresh Siddhafc1edaf2009-04-20 13:02:27 -07002294
Joerg Roedel70733e02012-09-26 12:44:33 +02002295 irq_remapping_disable();
Suresh Siddhafc1edaf2009-04-20 13:02:27 -07002296
Linus Torvalds1da177e2005-04-16 15:20:36 -07002297 local_irq_restore(flags);
2298 return 0;
2299}
2300
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01002301static void lapic_resume(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002302{
2303 unsigned int l, h;
2304 unsigned long flags;
Suresh Siddha31dce142011-05-18 16:31:33 -07002305 int maxlvt;
Fenghua Yub24696b2009-03-27 14:22:44 -07002306
Linus Torvalds1da177e2005-04-16 15:20:36 -07002307 if (!apic_pm_state.active)
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01002308 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002309
Fenghua Yub24696b2009-03-27 14:22:44 -07002310 local_irq_save(flags);
Joerg Roedel336224b2012-09-26 12:44:34 +02002311
2312 /*
2313 * IO-APIC and PIC have their own resume routines.
2314 * We just mask them here to make sure the interrupt
2315 * subsystem is completely quiet while we enable x2apic
2316 * and interrupt-remapping.
2317 */
2318 mask_ioapic_entries();
2319 legacy_pic->mask_all();
Karsten Wiesef990fff2006-12-07 02:14:11 +01002320
Thomas Gleixner659006b2015-01-15 21:22:26 +00002321 if (x2apic_mode) {
2322 __x2apic_enable();
2323 } else {
Cyrill Gorcunov92206c92008-08-16 23:21:51 +04002324 /*
2325 * Make sure the APICBASE points to the right address
2326 *
2327 * FIXME! This will be wrong if we ever support suspend on
2328 * SMP! We'll need to do this as part of the CPU restore!
2329 */
Bryan O'Donoghuecbf28292012-04-18 17:37:39 +01002330 if (boot_cpu_data.x86 >= 6) {
2331 rdmsr(MSR_IA32_APICBASE, l, h);
2332 l &= ~MSR_IA32_APICBASE_BASE;
2333 l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2334 wrmsr(MSR_IA32_APICBASE, l, h);
2335 }
Yinghai Lud5e629a2008-08-17 21:12:27 -07002336 }
Suresh Siddha6e1cb382008-07-10 11:16:58 -07002337
Fenghua Yub24696b2009-03-27 14:22:44 -07002338 maxlvt = lapic_get_maxlvt();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002339 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2340 apic_write(APIC_ID, apic_pm_state.apic_id);
2341 apic_write(APIC_DFR, apic_pm_state.apic_dfr);
2342 apic_write(APIC_LDR, apic_pm_state.apic_ldr);
2343 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
2344 apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
2345 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
2346 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
Paul Bolle71c69f72013-05-29 20:57:30 +02002347#if defined(CONFIG_X86_MCE_INTEL)
Karsten Wiesef990fff2006-12-07 02:14:11 +01002348 if (maxlvt >= 5)
2349 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
2350#endif
2351 if (maxlvt >= 4)
2352 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002353 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
2354 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
2355 apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
2356 apic_write(APIC_ESR, 0);
2357 apic_read(APIC_ESR);
2358 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
2359 apic_write(APIC_ESR, 0);
2360 apic_read(APIC_ESR);
Cyrill Gorcunov92206c92008-08-16 23:21:51 +04002361
Joerg Roedel70733e02012-09-26 12:44:33 +02002362 irq_remapping_reenable(x2apic_mode);
Suresh Siddha31dce142011-05-18 16:31:33 -07002363
Linus Torvalds1da177e2005-04-16 15:20:36 -07002364 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002365}
2366
Cyrill Gorcunov274cfe52008-08-16 23:21:53 +04002367/*
2368 * This device has no shutdown method - fully functioning local APICs
2369 * are needed on every CPU up until machine_halt/restart/poweroff.
2370 */
2371
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01002372static struct syscore_ops lapic_syscore_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002373 .resume = lapic_resume,
2374 .suspend = lapic_suspend,
2375};
2376
Paul Gortmaker148f9bb2013-06-18 18:23:59 -04002377static void apic_pm_activate(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002378{
2379 apic_pm_state.active = 1;
2380}
2381
2382static int __init init_lapic_sysfs(void)
2383{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002384 /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01002385 if (cpu_has_apic)
2386 register_syscore_ops(&lapic_syscore_ops);
Hiroshi Shimamotoe83a5fd2008-01-30 13:32:35 +01002387
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01002388 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002389}
Fenghua Yub24696b2009-03-27 14:22:44 -07002390
2391/* local apic needs to resume before other devices access its registers. */
2392core_initcall(init_lapic_sysfs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002393
2394#else /* CONFIG_PM */
2395
2396static void apic_pm_activate(void) { }
2397
2398#endif /* CONFIG_PM */
2399
Yinghai Luf28c0ae2008-08-24 02:01:49 -07002400#ifdef CONFIG_X86_64
Yinghai Lue0e42142009-04-26 23:39:38 -07002401
Paul Gortmaker148f9bb2013-06-18 18:23:59 -04002402static int multi_checked;
2403static int multi;
Yinghai Lue0e42142009-04-26 23:39:38 -07002404
Paul Gortmaker148f9bb2013-06-18 18:23:59 -04002405static int set_multi(const struct dmi_system_id *d)
Yinghai Lue0e42142009-04-26 23:39:38 -07002406{
2407 if (multi)
2408 return 0;
Cyrill Gorcunov6f0aced2009-05-01 23:54:25 +04002409 pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
Yinghai Lue0e42142009-04-26 23:39:38 -07002410 multi = 1;
2411 return 0;
2412}
2413
Paul Gortmaker148f9bb2013-06-18 18:23:59 -04002414static const struct dmi_system_id multi_dmi_table[] = {
Yinghai Lue0e42142009-04-26 23:39:38 -07002415 {
2416 .callback = set_multi,
2417 .ident = "IBM System Summit2",
2418 .matches = {
2419 DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2420 DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2421 },
2422 },
2423 {}
2424};
2425
Paul Gortmaker148f9bb2013-06-18 18:23:59 -04002426static void dmi_check_multi(void)
Yinghai Lue0e42142009-04-26 23:39:38 -07002427{
2428 if (multi_checked)
2429 return;
2430
2431 dmi_check_system(multi_dmi_table);
2432 multi_checked = 1;
2433}
2434
2435/*
2436 * apic_is_clustered_box() -- Check if we can expect good TSC
2437 *
2438 * Thus far, the major user of this is IBM's Summit2 series:
2439 * Clustered boxes may have unsynced TSC problems if they are
2440 * multi-chassis.
2441 * Use DMI to check them
2442 */
Paul Gortmaker148f9bb2013-06-18 18:23:59 -04002443int apic_is_clustered_box(void)
Yinghai Lue0e42142009-04-26 23:39:38 -07002444{
2445 dmi_check_multi();
Oren Twaig411cf9e2014-06-29 13:01:08 +03002446 return multi;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002447}
Yinghai Luf28c0ae2008-08-24 02:01:49 -07002448#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449
2450/*
Thomas Gleixner0e078e22008-01-30 13:30:20 +01002451 * APIC command line parameters
Linus Torvalds1da177e2005-04-16 15:20:36 -07002452 */
Cyrill Gorcunov789fa732008-08-18 20:46:01 +04002453static int __init setup_disableapic(char *arg)
Thomas Gleixner6935d1f2007-07-21 17:10:17 +02002454{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002455 disable_apic = 1;
Yinghai Lu9175fc02008-07-21 01:38:14 -07002456 setup_clear_cpu_cap(X86_FEATURE_APIC);
Andi Kleen2c8c0e62006-09-26 10:52:32 +02002457 return 0;
2458}
2459early_param("disableapic", setup_disableapic);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002460
Andi Kleen2c8c0e62006-09-26 10:52:32 +02002461/* same as disableapic, for compatibility */
Cyrill Gorcunov789fa732008-08-18 20:46:01 +04002462static int __init setup_nolapic(char *arg)
Thomas Gleixner6935d1f2007-07-21 17:10:17 +02002463{
Cyrill Gorcunov789fa732008-08-18 20:46:01 +04002464 return setup_disableapic(arg);
Thomas Gleixner6935d1f2007-07-21 17:10:17 +02002465}
Andi Kleen2c8c0e62006-09-26 10:52:32 +02002466early_param("nolapic", setup_nolapic);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002467
Linus Torvalds2e7c2832007-03-23 11:32:31 -07002468static int __init parse_lapic_timer_c2_ok(char *arg)
2469{
2470 local_apic_timer_c2_ok = 1;
2471 return 0;
2472}
2473early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2474
Cyrill Gorcunov36fef092008-08-15 13:51:20 +02002475static int __init parse_disable_apic_timer(char *arg)
Thomas Gleixner6935d1f2007-07-21 17:10:17 +02002476{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002477 disable_apic_timer = 1;
Cyrill Gorcunov36fef092008-08-15 13:51:20 +02002478 return 0;
Thomas Gleixner6935d1f2007-07-21 17:10:17 +02002479}
Cyrill Gorcunov36fef092008-08-15 13:51:20 +02002480early_param("noapictimer", parse_disable_apic_timer);
2481
2482static int __init parse_nolapic_timer(char *arg)
2483{
2484 disable_apic_timer = 1;
2485 return 0;
2486}
2487early_param("nolapic_timer", parse_nolapic_timer);
Andi Kleen73dea472006-02-03 21:50:50 +01002488
Cyrill Gorcunov79af9be2008-08-18 20:46:00 +04002489static int __init apic_set_verbosity(char *arg)
2490{
2491 if (!arg) {
2492#ifdef CONFIG_X86_64
2493 skip_ioapic_setup = 0;
Cyrill Gorcunov79af9be2008-08-18 20:46:00 +04002494 return 0;
2495#endif
2496 return -EINVAL;
2497 }
2498
2499 if (strcmp("debug", arg) == 0)
2500 apic_verbosity = APIC_DEBUG;
2501 else if (strcmp("verbose", arg) == 0)
2502 apic_verbosity = APIC_VERBOSE;
2503 else {
Cyrill Gorcunovba21ebb2008-11-10 09:16:41 +01002504 pr_warning("APIC Verbosity level %s not recognised"
Cyrill Gorcunov79af9be2008-08-18 20:46:00 +04002505 " use apic=verbose or apic=debug\n", arg);
2506 return -EINVAL;
2507 }
2508
2509 return 0;
2510}
2511early_param("apic", apic_set_verbosity);
2512
Yinghai Lu1e934dd2008-02-22 13:37:26 -08002513static int __init lapic_insert_resource(void)
2514{
2515 if (!apic_phys)
2516 return -1;
2517
2518 /* Put local APIC into the resource map. */
2519 lapic_resource.start = apic_phys;
2520 lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2521 insert_resource(&iomem_resource, &lapic_resource);
2522
2523 return 0;
2524}
2525
2526/*
2527 * need call insert after e820_reserve_resources()
2528 * that is using request_resource
2529 */
2530late_initcall(lapic_insert_resource);
HATAYAMA Daisuke151e0c72014-01-15 15:44:58 +09002531
2532static int __init apic_set_disabled_cpu_apicid(char *arg)
2533{
2534 if (!arg || !get_option(&arg, &disabled_cpu_apicid))
2535 return -EINVAL;
2536
2537 return 0;
2538}
2539early_param("disable_cpu_apicid", apic_set_disabled_cpu_apicid);