blob: 4640f6d64f8b406a636d60c4ea2263658dde5be6 [file] [log] [blame]
Paul Mackerras40ef8cb2005-10-10 22:50:37 +10001/*
2 *
3 * Common boot and setup code.
4 *
5 * Copyright (C) 2001 PPC64 Team, IBM Corp
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 */
12
Benjamin Herrenschmidt7191b612013-07-25 12:12:32 +100013#define DEBUG
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100014
Paul Gortmaker4b16f8e2011-07-22 18:24:23 -040015#include <linux/export.h>
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100016#include <linux/string.h>
17#include <linux/sched.h>
18#include <linux/init.h>
19#include <linux/kernel.h>
20#include <linux/reboot.h>
21#include <linux/delay.h>
22#include <linux/initrd.h>
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100023#include <linux/seq_file.h>
24#include <linux/ioport.h>
25#include <linux/console.h>
26#include <linux/utsname.h>
27#include <linux/tty.h>
28#include <linux/root_dev.h>
29#include <linux/notifier.h>
30#include <linux/cpu.h>
31#include <linux/unistd.h>
32#include <linux/serial.h>
33#include <linux/serial_8250.h>
Anton Blanchard7a0268f2006-01-11 13:16:44 +110034#include <linux/bootmem.h>
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110035#include <linux/pci.h>
Benjamin Herrenschmidt945feb12008-04-17 14:35:01 +100036#include <linux/lockdep.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100037#include <linux/memblock.h>
Anton Blancharda5d86252014-06-04 17:50:47 +100038#include <linux/memory.h>
Anton Blanchardc54b2bf2015-04-09 12:52:56 +100039#include <linux/nmi.h>
Becky Brucea6146882011-10-10 10:50:43 +000040
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100041#include <asm/io.h>
Michael Ellerman0cc47462005-12-04 18:39:37 +110042#include <asm/kdump.h>
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100043#include <asm/prom.h>
44#include <asm/processor.h>
45#include <asm/pgtable.h>
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100046#include <asm/smp.h>
47#include <asm/elf.h>
48#include <asm/machdep.h>
49#include <asm/paca.h>
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100050#include <asm/time.h>
51#include <asm/cputable.h>
Nicholas Piggin5a61ef72017-05-09 13:16:52 +100052#include <asm/dt_cpu_ftrs.h>
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100053#include <asm/sections.h>
54#include <asm/btext.h>
55#include <asm/nvram.h>
56#include <asm/setup.h>
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100057#include <asm/rtas.h>
58#include <asm/iommu.h>
59#include <asm/serial.h>
60#include <asm/cache.h>
61#include <asm/page.h>
62#include <asm/mmu.h>
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100063#include <asm/firmware.h>
Paul Mackerrasf78541d2005-10-28 22:53:37 +100064#include <asm/xmon.h>
David Gibsondcad47f2005-11-07 09:49:43 +110065#include <asm/udbg.h>
Michael Ellerman593e5372005-11-12 00:06:06 +110066#include <asm/kexec.h>
Kumar Galad36b4c42011-04-06 00:18:48 -050067#include <asm/code-patching.h>
Michael Ellerman5d31a962016-03-24 22:04:04 +110068#include <asm/livepatch.h>
Benjamin Herrenschmidtd3cbff12016-07-05 15:03:49 +100069#include <asm/opal.h>
Benjamin Herrenschmidtb1923ca2016-07-05 15:07:51 +100070#include <asm/cputhreads.h>
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100071
72#ifdef DEBUG
73#define DBG(fmt...) udbg_printf(fmt)
74#else
75#define DBG(fmt...)
76#endif
77
Chen Gang8246aca2013-03-20 14:30:12 +080078int spinning_secondaries;
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100079u64 ppc64_pft_size;
80
Olof Johanssondabcafd2005-12-08 19:40:17 -060081struct ppc64_caches ppc64_caches = {
Benjamin Herrenschmidte2827fe2017-01-08 17:31:47 -060082 .l1d = {
83 .block_size = 0x40,
84 .log_block_size = 6,
85 },
86 .l1i = {
87 .block_size = 0x40,
88 .log_block_size = 6
89 },
Olof Johanssondabcafd2005-12-08 19:40:17 -060090};
Paul Mackerras40ef8cb2005-10-10 22:50:37 +100091EXPORT_SYMBOL_GPL(ppc64_caches);
92
Scott Wood28efc352013-10-11 19:22:38 -050093#if defined(CONFIG_PPC_BOOK3E) && defined(CONFIG_SMP)
Benjamin Herrenschmidtb1923ca2016-07-05 15:07:51 +100094void __init setup_tlb_core_data(void)
Scott Wood28efc352013-10-11 19:22:38 -050095{
96 int cpu;
97
Scott Wood82d86de2014-03-07 14:48:35 -060098 BUILD_BUG_ON(offsetof(struct tlb_core_data, lock) != 0);
99
Scott Wood28efc352013-10-11 19:22:38 -0500100 for_each_possible_cpu(cpu) {
101 int first = cpu_first_thread_sibling(cpu);
102
Scott Woodd9e18312015-10-06 22:48:09 -0500103 /*
104 * If we boot via kdump on a non-primary thread,
105 * make sure we point at the thread that actually
106 * set up this TLB.
107 */
108 if (cpu_first_thread_sibling(boot_cpuid) == first)
109 first = boot_cpuid;
110
Scott Wood28efc352013-10-11 19:22:38 -0500111 paca[cpu].tcd_ptr = &paca[first].tcd;
112
113 /*
114 * If we have threads, we need either tlbsrx.
115 * or e6500 tablewalk mode, or else TLB handlers
116 * will be racy and could produce duplicate entries.
Michael Ellerman0d2b5cd2017-02-15 20:24:25 +1100117 * Should we panic instead?
Scott Wood28efc352013-10-11 19:22:38 -0500118 */
Michael Ellerman0d2b5cd2017-02-15 20:24:25 +1100119 WARN_ONCE(smt_enabled_at_boot >= 2 &&
120 !mmu_has_feature(MMU_FTR_USE_TLBRSRV) &&
121 book3e_htw_mode != PPC_HTW_E6500,
122 "%s: unsupported MMU configuration\n", __func__);
Scott Wood28efc352013-10-11 19:22:38 -0500123 }
124}
Scott Wood28efc352013-10-11 19:22:38 -0500125#endif
126
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000127#ifdef CONFIG_SMP
128
Nathan Fontenot954e6da2010-08-05 07:42:11 +0000129static char *smt_enabled_cmdline;
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000130
131/* Look for ibm,smt-enabled OF option */
Benjamin Herrenschmidtb1923ca2016-07-05 15:07:51 +1000132void __init check_smt_enabled(void)
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000133{
134 struct device_node *dn;
Jeremy Kerra7f67bd2006-07-12 15:35:54 +1000135 const char *smt_option;
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000136
Nathan Fontenot954e6da2010-08-05 07:42:11 +0000137 /* Default to enabling all threads */
138 smt_enabled_at_boot = threads_per_core;
139
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000140 /* Allow the command line to overrule the OF option */
Nathan Fontenot954e6da2010-08-05 07:42:11 +0000141 if (smt_enabled_cmdline) {
142 if (!strcmp(smt_enabled_cmdline, "on"))
143 smt_enabled_at_boot = threads_per_core;
144 else if (!strcmp(smt_enabled_cmdline, "off"))
145 smt_enabled_at_boot = 0;
146 else {
Daniel Walter1618bd52014-08-08 14:24:01 -0700147 int smt;
Nathan Fontenot954e6da2010-08-05 07:42:11 +0000148 int rc;
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000149
Daniel Walter1618bd52014-08-08 14:24:01 -0700150 rc = kstrtoint(smt_enabled_cmdline, 10, &smt);
Nathan Fontenot954e6da2010-08-05 07:42:11 +0000151 if (!rc)
152 smt_enabled_at_boot =
Daniel Walter1618bd52014-08-08 14:24:01 -0700153 min(threads_per_core, smt);
Nathan Fontenot954e6da2010-08-05 07:42:11 +0000154 }
155 } else {
156 dn = of_find_node_by_path("/options");
157 if (dn) {
158 smt_option = of_get_property(dn, "ibm,smt-enabled",
159 NULL);
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000160
Nathan Fontenot954e6da2010-08-05 07:42:11 +0000161 if (smt_option) {
162 if (!strcmp(smt_option, "on"))
163 smt_enabled_at_boot = threads_per_core;
164 else if (!strcmp(smt_option, "off"))
165 smt_enabled_at_boot = 0;
166 }
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000167
Nathan Fontenot954e6da2010-08-05 07:42:11 +0000168 of_node_put(dn);
169 }
170 }
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000171}
172
173/* Look for smt-enabled= cmdline option */
174static int __init early_smt_enabled(char *p)
175{
Nathan Fontenot954e6da2010-08-05 07:42:11 +0000176 smt_enabled_cmdline = p;
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000177 return 0;
178}
179early_param("smt-enabled", early_smt_enabled);
180
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000181#endif /* CONFIG_SMP */
182
Michael Ellerman25e13812013-02-12 14:44:50 +0000183/** Fix up paca fields required for the boot cpu */
Benjamin Herrenschmidt009776b2016-07-05 15:07:50 +1000184static void __init fixup_boot_paca(void)
Michael Ellerman25e13812013-02-12 14:44:50 +0000185{
186 /* The boot cpu is started */
187 get_paca()->cpu_start = 1;
188 /* Allow percpu accesses to work until we setup percpu data */
189 get_paca()->data_offset = 0;
190}
191
Benjamin Herrenschmidt009776b2016-07-05 15:07:50 +1000192static void __init configure_exceptions(void)
Benjamin Herrenschmidtd3cbff12016-07-05 15:03:49 +1000193{
194 /*
195 * Setup the trampolines from the lowmem exception vectors
196 * to the kdump kernel when not using a relocatable kernel.
197 */
198 setup_kdump_trampoline();
199
200 /* Under a PAPR hypervisor, we need hypercalls */
201 if (firmware_has_feature(FW_FEATURE_SET_MODE)) {
202 /* Enable AIL if possible */
203 pseries_enable_reloc_on_exc();
204
205 /*
206 * Tell the hypervisor that we want our exceptions to
207 * be taken in little endian mode.
208 *
209 * We don't call this for big endian as our calling convention
210 * makes us always enter in BE, and the call may fail under
211 * some circumstances with kdump.
212 */
213#ifdef __LITTLE_ENDIAN__
214 pseries_little_endian_exceptions();
215#endif
216 } else {
217 /* Set endian mode using OPAL */
218 if (firmware_has_feature(FW_FEATURE_OPAL))
219 opal_configure_cores();
220
Benjamin Herrenschmidtc0a36012016-11-15 15:28:33 +1100221 /* AIL on native is done in cpu_ready_for_interrupts() */
Benjamin Herrenschmidtd3cbff12016-07-05 15:03:49 +1000222 }
223}
224
Benjamin Herrenschmidt8f619b542014-03-28 13:36:30 +1100225static void cpu_ready_for_interrupts(void)
226{
Benjamin Herrenschmidtc0a36012016-11-15 15:28:33 +1100227 /*
228 * Enable AIL if supported, and we are in hypervisor mode. This
229 * is called once for every processor.
230 *
231 * If we are not in hypervisor mode the job is done once for
232 * the whole partition in configure_exceptions().
233 */
Michael Ellerman5511a452017-03-21 16:24:38 +1100234 if (cpu_has_feature(CPU_FTR_HVMODE) &&
235 cpu_has_feature(CPU_FTR_ARCH_207S)) {
Benjamin Herrenschmidtc0a36012016-11-15 15:28:33 +1100236 unsigned long lpcr = mfspr(SPRN_LPCR);
237 mtspr(SPRN_LPCR, lpcr | LPCR_AIL_3);
238 }
239
Benjamin Herrenschmidt7ed23e12017-03-20 17:49:03 +1100240 /*
241 * Fixup HFSCR:TM based on CPU features. The bit is set by our
242 * early asm init because at that point we haven't updated our
243 * CPU features from firmware and device-tree. Here we have,
244 * so let's do it.
245 */
246 if (cpu_has_feature(CPU_FTR_HVMODE) && !cpu_has_feature(CPU_FTR_TM_COMP))
247 mtspr(SPRN_HFSCR, mfspr(SPRN_HFSCR) & ~HFSCR_TM);
248
Benjamin Herrenschmidt8f619b542014-03-28 13:36:30 +1100249 /* Set IR and DR in PACA MSR */
250 get_paca()->kernel_msr = MSR_KERNEL;
Benjamin Herrenschmidt8f619b542014-03-28 13:36:30 +1100251}
252
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000253/*
254 * Early initialization entry point. This is called by head.S
255 * with MMU translation disabled. We rely on the "feature" of
256 * the CPU that ignores the top 2 bits of the address in real
257 * mode so we can access kernel globals normally provided we
258 * only toy with things in the RMO region. From here, we do
Yinghai Lu95f72d12010-07-12 14:36:09 +1000259 * some early parsing of the device-tree to setup out MEMBLOCK
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000260 * data structures, and allocate & initialize the hash table
261 * and segment tables so we can start running with translation
262 * enabled.
263 *
264 * It is this function which will call the probe() callback of
265 * the various platform types and copy the matching one to the
266 * global ppc_md structure. Your platform can eventually do
267 * some very early initializations from the probe() routine, but
268 * this is not recommended, be very careful as, for example, the
269 * device-tree is not accessible via normal means at this point.
270 */
271
272void __init early_setup(unsigned long dt_ptr)
273{
Geoff Levand6a7e4062013-02-13 17:03:16 +0000274 static __initdata struct paca_struct boot_paca;
275
Benjamin Herrenschmidt24d96492008-05-07 10:00:56 +1000276 /* -------- printk is _NOT_ safe to use here ! ------- */
277
Nicholas Piggin5a61ef72017-05-09 13:16:52 +1000278 /* Try new device tree based feature discovery ... */
279 if (!dt_cpu_ftrs_init(__va(dt_ptr)))
280 /* Otherwise use the old style CPU table */
281 identify_cpu(0, mfspr(SPRN_PVR));
Benjamin Herrenschmidt42c4aaa2006-10-24 16:42:40 +1000282
Michael Ellerman33dbcf72006-06-28 13:18:53 +1000283 /* Assume we're on cpu 0 for now. Don't write to the paca yet! */
Michael Ellerman1426d5a2010-01-28 13:23:22 +0000284 initialise_paca(&boot_paca, 0);
285 setup_paca(&boot_paca);
Michael Ellerman25e13812013-02-12 14:44:50 +0000286 fixup_boot_paca();
Michael Ellerman33dbcf72006-06-28 13:18:53 +1000287
Benjamin Herrenschmidt24d96492008-05-07 10:00:56 +1000288 /* -------- printk is now safe to use ------- */
289
Benjamin Herrenschmidtf2fd2512008-05-07 10:25:34 +1000290 /* Enable early debugging if any specified (see udbg.h) */
291 udbg_early_init();
292
Benjamin Herrenschmidte8222502006-03-28 23:15:54 +1100293 DBG(" -> early_setup(), dt_ptr: 0x%lx\n", dt_ptr);
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000294
295 /*
Linas Vepstas3c607ce2007-09-07 03:47:29 +1000296 * Do early initialization using the flattened device
297 * tree, such as retrieving the physical memory map or
298 * calculating/retrieving the hash table size.
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000299 */
300 early_init_devtree(__va(dt_ptr));
301
Anton Blanchard4df20462006-03-25 17:25:17 +1100302 /* Now we know the logical id of our boot cpu, setup the paca. */
Michael Ellerman1426d5a2010-01-28 13:23:22 +0000303 setup_paca(&paca[boot_cpuid]);
Michael Ellerman25e13812013-02-12 14:44:50 +0000304 fixup_boot_paca();
Anton Blanchard4df20462006-03-25 17:25:17 +1100305
Benjamin Herrenschmidt63c254a2016-07-05 15:03:46 +1000306 /*
Benjamin Herrenschmidtd3cbff12016-07-05 15:03:49 +1000307 * Configure exception handlers. This include setting up trampolines
308 * if needed, setting exception endian mode, etc...
Benjamin Herrenschmidt63c254a2016-07-05 15:03:46 +1000309 */
Benjamin Herrenschmidtd3cbff12016-07-05 15:03:49 +1000310 configure_exceptions();
Michael Ellerman0cc47462005-12-04 18:39:37 +1100311
Benjamin Herrenschmidtc4bd6cb2016-07-05 15:03:42 +1000312 /* Apply all the dynamic patching */
313 apply_feature_fixups();
Benjamin Herrenschmidt97f6e0c2016-08-10 17:27:34 +1000314 setup_feature_keys();
Benjamin Herrenschmidtc4bd6cb2016-07-05 15:03:42 +1000315
Michael Ellerman9e8066f2016-07-26 21:55:48 +1000316 /* Initialize the hash table or TLB handling */
317 early_init_mmu();
318
Benjamin Herrenschmidta944a9c2014-03-28 13:36:29 +1100319 /*
320 * At this point, we can let interrupts switch to virtual mode
321 * (the MMU has been setup), so adjust the MSR in the PACA to
Benjamin Herrenschmidt8f619b542014-03-28 13:36:30 +1100322 * have IR and DR set and enable AIL if it exists
Benjamin Herrenschmidta944a9c2014-03-28 13:36:29 +1100323 */
Benjamin Herrenschmidt8f619b542014-03-28 13:36:30 +1100324 cpu_ready_for_interrupts();
Benjamin Herrenschmidta944a9c2014-03-28 13:36:29 +1100325
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000326 DBG(" <- early_setup()\n");
Benjamin Herrenschmidt7191b612013-07-25 12:12:32 +1000327
328#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
329 /*
330 * This needs to be done *last* (after the above DBG() even)
331 *
332 * Right after we return from this function, we turn on the MMU
333 * which means the real-mode access trick that btext does will
334 * no longer work, it needs to switch to using a real MMU
335 * mapping. This call will ensure that it does
336 */
337 btext_map();
338#endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000339}
340
Paul Mackerras799d6042005-11-10 13:37:51 +1100341#ifdef CONFIG_SMP
342void early_setup_secondary(void)
343{
Madhavan Srinivasan103b7822016-03-04 10:31:48 +0530344 /* Mark interrupts disabled in PACA */
Benjamin Herrenschmidt757c74d2009-03-19 19:34:16 +0000345 get_paca()->soft_enabled = 0;
Paul Mackerras799d6042005-11-10 13:37:51 +1100346
Benjamin Herrenschmidt757c74d2009-03-19 19:34:16 +0000347 /* Initialize the hash table or TLB handling */
348 early_init_mmu_secondary();
Benjamin Herrenschmidta944a9c2014-03-28 13:36:29 +1100349
350 /*
351 * At this point, we can let interrupts switch to virtual mode
352 * (the MMU has been setup), so adjust the MSR in the PACA to
353 * have IR and DR set.
354 */
Benjamin Herrenschmidt8f619b542014-03-28 13:36:30 +1100355 cpu_ready_for_interrupts();
Paul Mackerras799d6042005-11-10 13:37:51 +1100356}
357
358#endif /* CONFIG_SMP */
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000359
Thiago Jung Bauermannda665882016-11-29 23:45:50 +1100360#if defined(CONFIG_SMP) || defined(CONFIG_KEXEC_CORE)
Scott Wood567cf942015-10-06 22:48:19 -0500361static bool use_spinloop(void)
362{
363 if (!IS_ENABLED(CONFIG_PPC_BOOK3E))
364 return true;
365
366 /*
367 * When book3e boots from kexec, the ePAPR spin table does
368 * not get used.
369 */
370 return of_property_read_bool(of_chosen, "linux,booted-from-kexec");
371}
372
Michael Ellermanb8f51022005-11-04 12:09:42 +1100373void smp_release_cpus(void)
374{
Michael Ellerman758438a2005-12-05 15:49:00 -0600375 unsigned long *ptr;
Benjamin Herrenschmidt9d07bc82011-03-16 14:54:35 +1100376 int i;
Michael Ellermanb8f51022005-11-04 12:09:42 +1100377
Scott Wood567cf942015-10-06 22:48:19 -0500378 if (!use_spinloop())
379 return;
380
Michael Ellermanb8f51022005-11-04 12:09:42 +1100381 DBG(" -> smp_release_cpus()\n");
382
383 /* All secondary cpus are spinning on a common spinloop, release them
384 * all now so they can start to spin on their individual paca
385 * spinloops. For non SMP kernels, the secondary cpus never get out
386 * of the common spinloop.
Paul Mackerras1f6a93e2008-08-30 11:40:24 +1000387 */
Michael Ellermanb8f51022005-11-04 12:09:42 +1100388
Michael Ellerman758438a2005-12-05 15:49:00 -0600389 ptr = (unsigned long *)((unsigned long)&__secondary_hold_spinloop
390 - PHYSICAL_START);
Anton Blanchard2751b622014-03-11 11:54:06 +1100391 *ptr = ppc_function_entry(generic_secondary_smp_init);
Benjamin Herrenschmidt9d07bc82011-03-16 14:54:35 +1100392
393 /* And wait a bit for them to catch up */
394 for (i = 0; i < 100000; i++) {
395 mb();
396 HMT_low();
Matt Evans7ac87ab2011-05-25 18:09:12 +0000397 if (spinning_secondaries == 0)
Benjamin Herrenschmidt9d07bc82011-03-16 14:54:35 +1100398 break;
399 udelay(1);
400 }
Matt Evans7ac87ab2011-05-25 18:09:12 +0000401 DBG("spinning_secondaries = %d\n", spinning_secondaries);
Michael Ellermanb8f51022005-11-04 12:09:42 +1100402
403 DBG(" <- smp_release_cpus()\n");
404}
Thiago Jung Bauermannda665882016-11-29 23:45:50 +1100405#endif /* CONFIG_SMP || CONFIG_KEXEC_CORE */
Michael Ellermanb8f51022005-11-04 12:09:42 +1100406
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000407/*
Paul Mackerras799d6042005-11-10 13:37:51 +1100408 * Initialize some remaining members of the ppc64_caches and systemcfg
409 * structures
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000410 * (at least until we get rid of them completely). This is mostly some
411 * cache informations about the CPU that will be used by cache flush
412 * routines and/or provided to userland
413 */
Benjamin Herrenschmidte2827fe2017-01-08 17:31:47 -0600414
415static void init_cache_info(struct ppc_cache_info *info, u32 size, u32 lsize,
416 u32 bsize, u32 sets)
417{
418 info->size = size;
419 info->sets = sets;
420 info->line_size = lsize;
421 info->block_size = bsize;
422 info->log_block_size = __ilog2(bsize);
Anton Blanchard6ba422c2017-03-05 10:54:34 +1100423 if (bsize)
424 info->blocks_per_page = PAGE_SIZE / bsize;
425 else
426 info->blocks_per_page = 0;
Benjamin Herrenschmidt98a5f362017-02-03 17:20:07 +1100427
428 if (sets == 0)
429 info->assoc = 0xffff;
430 else
431 info->assoc = size / (sets * lsize);
Benjamin Herrenschmidte2827fe2017-01-08 17:31:47 -0600432}
433
434static bool __init parse_cache_info(struct device_node *np,
435 bool icache,
436 struct ppc_cache_info *info)
437{
438 static const char *ipropnames[] __initdata = {
439 "i-cache-size",
440 "i-cache-sets",
441 "i-cache-block-size",
442 "i-cache-line-size",
443 };
444 static const char *dpropnames[] __initdata = {
445 "d-cache-size",
446 "d-cache-sets",
447 "d-cache-block-size",
448 "d-cache-line-size",
449 };
450 const char **propnames = icache ? ipropnames : dpropnames;
451 const __be32 *sizep, *lsizep, *bsizep, *setsp;
452 u32 size, lsize, bsize, sets;
453 bool success = true;
454
455 size = 0;
456 sets = -1u;
457 lsize = bsize = cur_cpu_spec->dcache_bsize;
458 sizep = of_get_property(np, propnames[0], NULL);
459 if (sizep != NULL)
460 size = be32_to_cpu(*sizep);
461 setsp = of_get_property(np, propnames[1], NULL);
462 if (setsp != NULL)
463 sets = be32_to_cpu(*setsp);
464 bsizep = of_get_property(np, propnames[2], NULL);
465 lsizep = of_get_property(np, propnames[3], NULL);
466 if (bsizep == NULL)
467 bsizep = lsizep;
468 if (lsizep != NULL)
469 lsize = be32_to_cpu(*lsizep);
470 if (bsizep != NULL)
471 bsize = be32_to_cpu(*bsizep);
472 if (sizep == NULL || bsizep == NULL || lsizep == NULL)
473 success = false;
474
475 /*
476 * OF is weird .. it represents fully associative caches
477 * as "1 way" which doesn't make much sense and doesn't
478 * leave room for direct mapped. We'll assume that 0
479 * in OF means direct mapped for that reason.
480 */
481 if (sets == 1)
482 sets = 0;
483 else if (sets == 0)
484 sets = 1;
485
486 init_cache_info(info, size, lsize, bsize, sets);
487
488 return success;
489}
490
Benjamin Herrenschmidtb1923ca2016-07-05 15:07:51 +1000491void __init initialize_cache_info(void)
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000492{
Benjamin Herrenschmidt608b4212017-01-08 17:31:49 -0600493 struct device_node *cpu = NULL, *l2, *l3 = NULL;
494 u32 pvr;
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000495
496 DBG(" -> initialize_cache_info()\n");
497
Benjamin Herrenschmidt608b4212017-01-08 17:31:49 -0600498 /*
499 * All shipping POWER8 machines have a firmware bug that
500 * puts incorrect information in the device-tree. This will
501 * be (hopefully) fixed for future chips but for now hard
502 * code the values if we are running on one of these
503 */
504 pvr = PVR_VER(mfspr(SPRN_PVR));
505 if (pvr == PVR_POWER8 || pvr == PVR_POWER8E ||
506 pvr == PVR_POWER8NVL) {
507 /* size lsize blk sets */
508 init_cache_info(&ppc64_caches.l1i, 0x8000, 128, 128, 32);
509 init_cache_info(&ppc64_caches.l1d, 0x10000, 128, 128, 64);
510 init_cache_info(&ppc64_caches.l2, 0x80000, 128, 0, 512);
511 init_cache_info(&ppc64_caches.l3, 0x800000, 128, 0, 8192);
512 } else
513 cpu = of_find_node_by_type(NULL, "cpu");
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000514
Benjamin Herrenschmidte2827fe2017-01-08 17:31:47 -0600515 /*
516 * We're assuming *all* of the CPUs have the same
517 * d-cache and i-cache sizes... -Peter
518 */
Benjamin Herrenschmidt65e01f32017-01-08 17:31:48 -0600519 if (cpu) {
520 if (!parse_cache_info(cpu, false, &ppc64_caches.l1d))
Benjamin Herrenschmidte2827fe2017-01-08 17:31:47 -0600521 DBG("Argh, can't find dcache properties !\n");
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000522
Benjamin Herrenschmidt65e01f32017-01-08 17:31:48 -0600523 if (!parse_cache_info(cpu, true, &ppc64_caches.l1i))
Benjamin Herrenschmidte2827fe2017-01-08 17:31:47 -0600524 DBG("Argh, can't find icache properties !\n");
Benjamin Herrenschmidt65e01f32017-01-08 17:31:48 -0600525
526 /*
527 * Try to find the L2 and L3 if any. Assume they are
528 * unified and use the D-side properties.
529 */
530 l2 = of_find_next_cache_node(cpu);
531 of_node_put(cpu);
532 if (l2) {
533 parse_cache_info(l2, false, &ppc64_caches.l2);
534 l3 = of_find_next_cache_node(l2);
535 of_node_put(l2);
536 }
537 if (l3) {
538 parse_cache_info(l3, false, &ppc64_caches.l3);
539 of_node_put(l3);
540 }
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000541 }
542
Benjamin Herrenschmidt9df549a2016-07-05 15:04:08 +1000543 /* For use by binfmt_elf */
Benjamin Herrenschmidte2827fe2017-01-08 17:31:47 -0600544 dcache_bsize = ppc64_caches.l1d.block_size;
545 icache_bsize = ppc64_caches.l1i.block_size;
Benjamin Herrenschmidt9df549a2016-07-05 15:04:08 +1000546
Nicholas Piggin5a61ef72017-05-09 13:16:52 +1000547 cur_cpu_spec->dcache_bsize = dcache_bsize;
548 cur_cpu_spec->icache_bsize = icache_bsize;
549
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000550 DBG(" <- initialize_cache_info()\n");
551}
552
Benjamin Herrenschmidt40bd5872011-05-03 14:07:01 +0000553/* This returns the limit below which memory accesses to the linear
554 * mapping are guarnateed not to cause a TLB or SLB miss. This is
555 * used to allocate interrupt or emergency stacks for which our
556 * exception entry path doesn't deal with being interrupted.
557 */
Benjamin Herrenschmidt009776b2016-07-05 15:07:50 +1000558static __init u64 safe_stack_limit(void)
Anton Blanchard095c7962010-05-10 18:59:18 +0000559{
Benjamin Herrenschmidt40bd5872011-05-03 14:07:01 +0000560#ifdef CONFIG_PPC_BOOK3E
561 /* Freescale BookE bolts the entire linear mapping */
562 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E))
563 return linear_map_top;
564 /* Other BookE, we assume the first GB is bolted */
565 return 1ul << 30;
566#else
567 /* BookS, the first segment is bolted */
568 if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
Anton Blanchard095c7962010-05-10 18:59:18 +0000569 return 1UL << SID_SHIFT_1T;
Anton Blanchard095c7962010-05-10 18:59:18 +0000570 return 1UL << SID_SHIFT;
Benjamin Herrenschmidt40bd5872011-05-03 14:07:01 +0000571#endif
Anton Blanchard095c7962010-05-10 18:59:18 +0000572}
573
Benjamin Herrenschmidtb1923ca2016-07-05 15:07:51 +1000574void __init irqstack_early_init(void)
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000575{
Benjamin Herrenschmidt40bd5872011-05-03 14:07:01 +0000576 u64 limit = safe_stack_limit();
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000577 unsigned int i;
578
579 /*
Anton Blanchard8f4da262010-12-08 00:55:03 +0000580 * Interrupt stacks must be in the first segment since we
581 * cannot afford to take SLB misses on them.
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000582 */
KAMEZAWA Hiroyuki0e551952006-03-28 14:50:51 -0800583 for_each_possible_cpu(i) {
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100584 softirq_ctx[i] = (struct thread_info *)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000585 __va(memblock_alloc_base(THREAD_SIZE,
Anton Blanchard095c7962010-05-10 18:59:18 +0000586 THREAD_SIZE, limit));
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100587 hardirq_ctx[i] = (struct thread_info *)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000588 __va(memblock_alloc_base(THREAD_SIZE,
Anton Blanchard095c7962010-05-10 18:59:18 +0000589 THREAD_SIZE, limit));
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000590 }
591}
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000592
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000593#ifdef CONFIG_PPC_BOOK3E
Benjamin Herrenschmidtb1923ca2016-07-05 15:07:51 +1000594void __init exc_lvl_early_init(void)
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000595{
596 unsigned int i;
Tiejun Chen160c7322013-10-23 17:31:21 +0800597 unsigned long sp;
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000598
599 for_each_possible_cpu(i) {
Tiejun Chen160c7322013-10-23 17:31:21 +0800600 sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE);
601 critirq_ctx[i] = (struct thread_info *)__va(sp);
602 paca[i].crit_kstack = __va(sp + THREAD_SIZE);
603
604 sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE);
605 dbgirq_ctx[i] = (struct thread_info *)__va(sp);
606 paca[i].dbg_kstack = __va(sp + THREAD_SIZE);
607
608 sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE);
609 mcheckirq_ctx[i] = (struct thread_info *)__va(sp);
610 paca[i].mc_kstack = __va(sp + THREAD_SIZE);
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000611 }
Kumar Galad36b4c42011-04-06 00:18:48 -0500612
613 if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC))
Kevin Hao565c2f22013-05-12 07:26:23 +0800614 patch_exception(0x040, exc_debug_debug_book3e);
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000615}
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000616#endif
617
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000618/*
Nicholas Piggin34f19ff2017-06-21 15:58:29 +1000619 * Emergency stacks are used for a range of things, from asynchronous
620 * NMIs (system reset, machine check) to synchronous, process context.
621 * We set preempt_count to zero, even though that isn't necessarily correct. To
622 * get the right value we'd need to copy it from the previous thread_info, but
623 * doing that might fault causing more problems.
624 * TODO: what to do with accounting?
625 */
626static void emerg_stack_init_thread_info(struct thread_info *ti, int cpu)
627{
628 ti->task = NULL;
629 ti->cpu = cpu;
630 ti->preempt_count = 0;
631 ti->local_flags = 0;
632 ti->flags = 0;
633 klp_init_thread_info(ti);
634}
635
636/*
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000637 * Stack space used when we detect a bad kernel stack pointer, and
Mahesh Salgaonkar729b0f72013-10-30 20:04:00 +0530638 * early in SMP boots before relocation is enabled. Exclusive emergency
639 * stack for machine checks.
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000640 */
Benjamin Herrenschmidtb1923ca2016-07-05 15:07:51 +1000641void __init emergency_stack_init(void)
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000642{
Anton Blanchard095c7962010-05-10 18:59:18 +0000643 u64 limit;
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000644 unsigned int i;
645
646 /*
647 * Emergency stacks must be under 256MB, we cannot afford to take
648 * SLB misses on them. The ABI also requires them to be 128-byte
649 * aligned.
650 *
651 * Since we use these as temporary stacks during secondary CPU
652 * bringup, we need to get at them in real mode. This means they
653 * must also be within the RMO region.
Nicholas Piggin34f19ff2017-06-21 15:58:29 +1000654 *
655 * The IRQ stacks allocated elsewhere in this file are zeroed and
656 * initialized in kernel/irq.c. These are initialized here in order
657 * to have emergency stacks available as early as possible.
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000658 */
Benjamin Herrenschmidt40bd5872011-05-03 14:07:01 +0000659 limit = min(safe_stack_limit(), ppc64_rma_size);
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000660
Michael Ellerman3243d872008-04-30 13:21:45 +1000661 for_each_possible_cpu(i) {
Michael Ellerman5d31a962016-03-24 22:04:04 +1100662 struct thread_info *ti;
663 ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit));
Nicholas Piggin34f19ff2017-06-21 15:58:29 +1000664 memset(ti, 0, THREAD_SIZE);
665 emerg_stack_init_thread_info(ti, i);
Michael Ellerman5d31a962016-03-24 22:04:04 +1100666 paca[i].emergency_sp = (void *)ti + THREAD_SIZE;
Mahesh Salgaonkar729b0f72013-10-30 20:04:00 +0530667
668#ifdef CONFIG_PPC_BOOK3S_64
Nicholas Pigginb1ee8a32016-12-20 04:30:06 +1000669 /* emergency stack for NMI exception handling. */
670 ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit));
Nicholas Piggin34f19ff2017-06-21 15:58:29 +1000671 memset(ti, 0, THREAD_SIZE);
672 emerg_stack_init_thread_info(ti, i);
Nicholas Pigginb1ee8a32016-12-20 04:30:06 +1000673 paca[i].nmi_emergency_sp = (void *)ti + THREAD_SIZE;
674
Mahesh Salgaonkar729b0f72013-10-30 20:04:00 +0530675 /* emergency stack for machine check exception handling. */
Michael Ellerman5d31a962016-03-24 22:04:04 +1100676 ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit));
Nicholas Piggin34f19ff2017-06-21 15:58:29 +1000677 memset(ti, 0, THREAD_SIZE);
678 emerg_stack_init_thread_info(ti, i);
Michael Ellerman5d31a962016-03-24 22:04:04 +1100679 paca[i].mc_emergency_sp = (void *)ti + THREAD_SIZE;
Mahesh Salgaonkar729b0f72013-10-30 20:04:00 +0530680#endif
Michael Ellerman3243d872008-04-30 13:21:45 +1000681 }
Paul Mackerras40ef8cb2005-10-10 22:50:37 +1000682}
683
Anton Blanchard7a0268f2006-01-11 13:16:44 +1100684#ifdef CONFIG_SMP
Tejun Heoc2a7e812009-08-14 15:00:53 +0900685#define PCPU_DYN_SIZE ()
686
687static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align)
688{
Michael Ellermanba4a6482017-06-06 20:23:57 +1000689 return __alloc_bootmem_node(NODE_DATA(early_cpu_to_node(cpu)), size, align,
Tejun Heoc2a7e812009-08-14 15:00:53 +0900690 __pa(MAX_DMA_ADDRESS));
691}
692
693static void __init pcpu_fc_free(void *ptr, size_t size)
694{
695 free_bootmem(__pa(ptr), size);
696}
697
698static int pcpu_cpu_distance(unsigned int from, unsigned int to)
699{
Michael Ellermanba4a6482017-06-06 20:23:57 +1000700 if (early_cpu_to_node(from) == early_cpu_to_node(to))
Tejun Heoc2a7e812009-08-14 15:00:53 +0900701 return LOCAL_DISTANCE;
702 else
703 return REMOTE_DISTANCE;
704}
705
Anton Blanchardae01f842010-05-31 18:45:11 +0000706unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
707EXPORT_SYMBOL(__per_cpu_offset);
708
Anton Blanchard7a0268f2006-01-11 13:16:44 +1100709void __init setup_per_cpu_areas(void)
710{
Tejun Heoc2a7e812009-08-14 15:00:53 +0900711 const size_t dyn_size = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE;
712 size_t atom_size;
713 unsigned long delta;
714 unsigned int cpu;
715 int rc;
Anton Blanchard7a0268f2006-01-11 13:16:44 +1100716
Tejun Heoc2a7e812009-08-14 15:00:53 +0900717 /*
718 * Linear mapping is one of 4K, 1M and 16M. For 4K, no need
719 * to group units. For larger mappings, use 1M atom which
720 * should be large enough to contain a number of units.
721 */
722 if (mmu_linear_psize == MMU_PAGE_4K)
723 atom_size = PAGE_SIZE;
724 else
725 atom_size = 1 << 20;
Anton Blanchard7a0268f2006-01-11 13:16:44 +1100726
Tejun Heoc2a7e812009-08-14 15:00:53 +0900727 rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance,
728 pcpu_fc_alloc, pcpu_fc_free);
729 if (rc < 0)
730 panic("cannot initialize percpu area (err=%d)", rc);
Anton Blanchard7a0268f2006-01-11 13:16:44 +1100731
Tejun Heoc2a7e812009-08-14 15:00:53 +0900732 delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
Anton Blanchardae01f842010-05-31 18:45:11 +0000733 for_each_possible_cpu(cpu) {
734 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
735 paca[cpu].data_offset = __per_cpu_offset[cpu];
736 }
Anton Blanchard7a0268f2006-01-11 13:16:44 +1100737}
738#endif
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100739
Anton Blancharda5d86252014-06-04 17:50:47 +1000740#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
741unsigned long memory_block_size_bytes(void)
742{
743 if (ppc_md.memory_block_size)
744 return ppc_md.memory_block_size();
745
746 return MIN_MEMORY_BLOCK_SIZE;
747}
748#endif
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100749
Benjamin Herrenschmidtecd73cc2013-07-15 13:03:08 +1000750#if defined(CONFIG_PPC_INDIRECT_PIO) || defined(CONFIG_PPC_INDIRECT_MMIO)
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100751struct ppc_pci_io ppc_pci_io;
752EXPORT_SYMBOL(ppc_pci_io);
Benjamin Herrenschmidtecd73cc2013-07-15 13:03:08 +1000753#endif
Anton Blanchardc54b2bf2015-04-09 12:52:56 +1000754
755#ifdef CONFIG_HARDLOCKUP_DETECTOR
756u64 hw_nmi_get_sample_period(int watchdog_thresh)
757{
758 return ppc_proc_freq * watchdog_thresh;
759}
760
761/*
762 * The hardlockup detector breaks PMU event based branches and is likely
763 * to get false positives in KVM guests, so disable it by default.
764 */
765static int __init disable_hardlockup_detector(void)
766{
Linus Torvaldsd19d5ef2015-04-16 13:53:32 -0500767 hardlockup_detector_disable();
Anton Blanchardc54b2bf2015-04-09 12:52:56 +1000768
769 return 0;
770}
771early_initcall(disable_hardlockup_detector);
772#endif