blob: 83a4ea6e3d605fb6637a845924887583a1a08910 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * arch/s390/kernel/smp.c
3 *
Heiko Carstens255acee2006-02-17 13:52:46 -08004 * Copyright (C) IBM Corp. 1999,2006
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
6 * Martin Schwidefsky (schwidefsky@de.ibm.com)
7 * Heiko Carstens (heiko.carstens@de.ibm.com)
8 *
9 * based on other smp stuff by
10 * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net>
11 * (c) 1998 Ingo Molnar
12 *
13 * We work with logical cpu numbering everywhere we can. The only
14 * functions using the real cpu address (got from STAP) are the sigp
15 * functions. For all other functions we use the identity mapping.
16 * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
17 * used e.g. to find the idle task belonging to a logical cpu. Every array
18 * in the kernel is sorted by the logical cpu number and not by the physical
19 * one which is causing all the confusion with __cpu_logical_map and
20 * cpu_number_map in other architectures.
21 */
22
23#include <linux/module.h>
24#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/mm.h>
26#include <linux/spinlock.h>
27#include <linux/kernel_stat.h>
28#include <linux/smp_lock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/delay.h>
30#include <linux/cache.h>
31#include <linux/interrupt.h>
32#include <linux/cpu.h>
Heiko Carstens2b67fc42007-02-05 21:16:47 +010033#include <linux/timex.h>
34#include <asm/setup.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <asm/sigp.h>
36#include <asm/pgalloc.h>
37#include <asm/irq.h>
38#include <asm/s390_ext.h>
39#include <asm/cpcmd.h>
40#include <asm/tlbflush.h>
Heiko Carstens2b67fc42007-02-05 21:16:47 +010041#include <asm/timer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
Linus Torvalds1da177e2005-04-16 15:20:36 -070043extern volatile int __cpu_logical_map[];
44
45/*
46 * An array with a pointer the lowcore of every CPU.
47 */
48
49struct _lowcore *lowcore_ptr[NR_CPUS];
50
Heiko Carstens255acee2006-02-17 13:52:46 -080051cpumask_t cpu_online_map = CPU_MASK_NONE;
52cpumask_t cpu_possible_map = CPU_MASK_NONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053
54static struct task_struct *current_set[NR_CPUS];
55
Linus Torvalds1da177e2005-04-16 15:20:36 -070056static void smp_ext_bitcall(int, ec_bit_sig);
57static void smp_ext_bitcall_others(ec_bit_sig);
58
59/*
Heiko Carstens0ec67662007-02-12 15:47:04 +010060 * Structure and data for smp_call_function(). This is designed to minimise
Linus Torvalds1da177e2005-04-16 15:20:36 -070061 * static memory requirements. It also looks cleaner.
62 */
63static DEFINE_SPINLOCK(call_lock);
64
65struct call_data_struct {
66 void (*func) (void *info);
67 void *info;
68 atomic_t started;
69 atomic_t finished;
70 int wait;
71};
72
73static struct call_data_struct * call_data;
74
75/*
76 * 'Call function' interrupt callback
77 */
78static void do_call_function(void)
79{
80 void (*func) (void *info) = call_data->func;
81 void *info = call_data->info;
82 int wait = call_data->wait;
83
84 atomic_inc(&call_data->started);
85 (*func)(info);
86 if (wait)
87 atomic_inc(&call_data->finished);
88}
89
90/*
91 * this function sends a 'generic call function' IPI to all other CPUs
92 * in the system.
93 */
94
95int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
96 int wait)
97/*
98 * [SUMMARY] Run a function on all other CPUs.
99 * <func> The function to run. This must be fast and non-blocking.
100 * <info> An arbitrary pointer to pass to the function.
101 * <nonatomic> currently unused.
102 * <wait> If true, wait (atomically) until function has completed on other CPUs.
103 * [RETURNS] 0 on success, else a negative status code. Does not return until
104 * remote CPUs are nearly ready to execute <<func>> or are or have executed.
105 *
106 * You must not call this function with disabled interrupts or from a
Heiko Carstens0ec67662007-02-12 15:47:04 +0100107 * hardware interrupt handler.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 */
109{
110 struct call_data_struct data;
111 int cpus = num_online_cpus()-1;
112
113 if (cpus <= 0)
114 return 0;
115
Heiko Carstens0ec67662007-02-12 15:47:04 +0100116 /* Can deadlock when interrupts are disabled or if in wrong context */
117 WARN_ON(irqs_disabled() || in_irq());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118
119 data.func = func;
120 data.info = info;
121 atomic_set(&data.started, 0);
122 data.wait = wait;
123 if (wait)
124 atomic_set(&data.finished, 0);
125
Heiko Carstens0ec67662007-02-12 15:47:04 +0100126 spin_lock_bh(&call_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127 call_data = &data;
128 /* Send a message to all other CPUs and wait for them to respond */
129 smp_ext_bitcall_others(ec_call_function);
130
131 /* Wait for response */
132 while (atomic_read(&data.started) != cpus)
133 cpu_relax();
134
135 if (wait)
136 while (atomic_read(&data.finished) != cpus)
137 cpu_relax();
Heiko Carstens0ec67662007-02-12 15:47:04 +0100138 spin_unlock_bh(&call_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
140 return 0;
141}
142
143/*
144 * Call a function on one CPU
145 * cpu : the CPU the function should be executed on
146 *
147 * You must not call this function with disabled interrupts or from a
148 * hardware interrupt handler. You may call it from a bottom half.
149 *
150 * It is guaranteed that the called function runs on the specified CPU,
151 * preemption is disabled.
152 */
153int smp_call_function_on(void (*func) (void *info), void *info,
154 int nonatomic, int wait, int cpu)
155{
156 struct call_data_struct data;
157 int curr_cpu;
158
159 if (!cpu_online(cpu))
160 return -EINVAL;
161
Heiko Carstens0ec67662007-02-12 15:47:04 +0100162 /* Can deadlock when interrupts are disabled or if in wrong context */
163 WARN_ON(irqs_disabled() || in_irq());
164
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 /* disable preemption for local function call */
166 curr_cpu = get_cpu();
167
168 if (curr_cpu == cpu) {
169 /* direct call to function */
170 func(info);
171 put_cpu();
172 return 0;
173 }
174
175 data.func = func;
176 data.info = info;
177 atomic_set(&data.started, 0);
178 data.wait = wait;
179 if (wait)
180 atomic_set(&data.finished, 0);
181
182 spin_lock_bh(&call_lock);
183 call_data = &data;
184 smp_ext_bitcall(cpu, ec_call_function);
185
186 /* Wait for response */
187 while (atomic_read(&data.started) != 1)
188 cpu_relax();
189
190 if (wait)
191 while (atomic_read(&data.finished) != 1)
192 cpu_relax();
193
194 spin_unlock_bh(&call_lock);
195 put_cpu();
196 return 0;
197}
198EXPORT_SYMBOL(smp_call_function_on);
199
Heiko Carstens4d284ca2007-02-05 21:18:53 +0100200static void do_send_stop(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201{
202 int cpu, rc;
203
204 /* stop all processors */
205 for_each_online_cpu(cpu) {
206 if (cpu == smp_processor_id())
207 continue;
208 do {
209 rc = signal_processor(cpu, sigp_stop);
210 } while (rc == sigp_busy);
211 }
212}
213
Heiko Carstens4d284ca2007-02-05 21:18:53 +0100214static void do_store_status(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215{
216 int cpu, rc;
217
218 /* store status of all processors in their lowcores (real 0) */
219 for_each_online_cpu(cpu) {
220 if (cpu == smp_processor_id())
221 continue;
222 do {
223 rc = signal_processor_p(
224 (__u32)(unsigned long) lowcore_ptr[cpu], cpu,
225 sigp_store_status_at_address);
226 } while(rc == sigp_busy);
227 }
228}
229
Heiko Carstens4d284ca2007-02-05 21:18:53 +0100230static void do_wait_for_stop(void)
Heiko Carstensc6b5b842006-12-04 15:40:33 +0100231{
232 int cpu;
233
234 /* Wait for all other cpus to enter stopped state */
235 for_each_online_cpu(cpu) {
236 if (cpu == smp_processor_id())
237 continue;
238 while(!smp_cpu_not_running(cpu))
239 cpu_relax();
240 }
241}
242
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243/*
244 * this function sends a 'stop' sigp to all other CPUs in the system.
245 * it goes straight through.
246 */
247void smp_send_stop(void)
248{
Heiko Carstensc6b5b842006-12-04 15:40:33 +0100249 /* Disable all interrupts/machine checks */
Gerald Schaeferc1821c22007-02-05 21:18:17 +0100250 __load_psw_mask(psw_kernel_bits & ~PSW_MASK_MCHECK);
Heiko Carstensc6b5b842006-12-04 15:40:33 +0100251
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252 /* write magic number to zero page (absolute 0) */
253 lowcore_ptr[smp_processor_id()]->panic_magic = __PANIC_MAGIC;
254
255 /* stop other processors. */
256 do_send_stop();
257
Heiko Carstensc6b5b842006-12-04 15:40:33 +0100258 /* wait until other processors are stopped */
259 do_wait_for_stop();
260
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261 /* store status of other processors. */
262 do_store_status();
263}
264
265/*
266 * Reboot, halt and power_off routines for SMP.
267 */
268
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269void machine_restart_smp(char * __unused)
270{
Heiko Carstensc6b5b842006-12-04 15:40:33 +0100271 smp_send_stop();
272 do_reipl();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273}
274
275void machine_halt_smp(void)
276{
Heiko Carstensc6b5b842006-12-04 15:40:33 +0100277 smp_send_stop();
278 if (MACHINE_IS_VM && strlen(vmhalt_cmd) > 0)
279 __cpcmd(vmhalt_cmd, NULL, 0, NULL);
280 signal_processor(smp_processor_id(), sigp_stop_and_store_status);
281 for (;;);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282}
283
284void machine_power_off_smp(void)
285{
Heiko Carstensc6b5b842006-12-04 15:40:33 +0100286 smp_send_stop();
287 if (MACHINE_IS_VM && strlen(vmpoff_cmd) > 0)
288 __cpcmd(vmpoff_cmd, NULL, 0, NULL);
289 signal_processor(smp_processor_id(), sigp_stop_and_store_status);
290 for (;;);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291}
292
293/*
294 * This is the main routine where commands issued by other
295 * cpus are handled.
296 */
297
Heiko Carstens2b67fc42007-02-05 21:16:47 +0100298static void do_ext_call_interrupt(__u16 code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299{
300 unsigned long bits;
301
302 /*
303 * handle bit signal external calls
304 *
305 * For the ec_schedule signal we have to do nothing. All the work
306 * is done automatically when we return from the interrupt.
307 */
308 bits = xchg(&S390_lowcore.ext_call_fast, 0);
309
310 if (test_bit(ec_call_function, &bits))
311 do_call_function();
312}
313
314/*
315 * Send an external call sigp to another cpu and return without waiting
316 * for its completion.
317 */
318static void smp_ext_bitcall(int cpu, ec_bit_sig sig)
319{
320 /*
321 * Set signaling bit in lowcore of target cpu and kick it
322 */
323 set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
Heiko Carstens99b2d8d2005-07-27 11:45:00 -0700324 while(signal_processor(cpu, sigp_emergency_signal) == sigp_busy)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 udelay(10);
326}
327
328/*
329 * Send an external call sigp to every other cpu in the system and
330 * return without waiting for its completion.
331 */
332static void smp_ext_bitcall_others(ec_bit_sig sig)
333{
334 int cpu;
335
336 for_each_online_cpu(cpu) {
337 if (cpu == smp_processor_id())
338 continue;
339 /*
340 * Set signaling bit in lowcore of target cpu and kick it
341 */
342 set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
Heiko Carstens99b2d8d2005-07-27 11:45:00 -0700343 while (signal_processor(cpu, sigp_emergency_signal) == sigp_busy)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 udelay(10);
345 }
346}
347
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -0800348#ifndef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349/*
350 * this function sends a 'purge tlb' signal to another CPU.
351 */
352void smp_ptlb_callback(void *info)
353{
354 local_flush_tlb();
355}
356
357void smp_ptlb_all(void)
358{
359 on_each_cpu(smp_ptlb_callback, NULL, 0, 1);
360}
361EXPORT_SYMBOL(smp_ptlb_all);
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -0800362#endif /* ! CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363
364/*
365 * this function sends a 'reschedule' IPI to another CPU.
366 * it goes straight through and wastes no time serializing
367 * anything. Worst case is that we lose a reschedule ...
368 */
369void smp_send_reschedule(int cpu)
370{
371 smp_ext_bitcall(cpu, ec_schedule);
372}
373
374/*
375 * parameter area for the set/clear control bit callbacks
376 */
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200377struct ec_creg_mask_parms {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 unsigned long orvals[16];
379 unsigned long andvals[16];
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200380};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381
382/*
383 * callback for setting/clearing control bits
384 */
Heiko Carstens2b67fc42007-02-05 21:16:47 +0100385static void smp_ctl_bit_callback(void *info) {
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200386 struct ec_creg_mask_parms *pp = info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 unsigned long cregs[16];
388 int i;
389
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200390 __ctl_store(cregs, 0, 15);
391 for (i = 0; i <= 15; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392 cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200393 __ctl_load(cregs, 0, 15);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394}
395
396/*
397 * Set a bit in a control register of all cpus
398 */
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200399void smp_ctl_set_bit(int cr, int bit)
400{
401 struct ec_creg_mask_parms parms;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200403 memset(&parms.orvals, 0, sizeof(parms.orvals));
404 memset(&parms.andvals, 0xff, sizeof(parms.andvals));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405 parms.orvals[cr] = 1 << bit;
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200406 on_each_cpu(smp_ctl_bit_callback, &parms, 0, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407}
408
409/*
410 * Clear a bit in a control register of all cpus
411 */
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200412void smp_ctl_clear_bit(int cr, int bit)
413{
414 struct ec_creg_mask_parms parms;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200416 memset(&parms.orvals, 0, sizeof(parms.orvals));
417 memset(&parms.andvals, 0xff, sizeof(parms.andvals));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418 parms.andvals[cr] = ~(1L << bit);
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200419 on_each_cpu(smp_ctl_bit_callback, &parms, 0, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420}
421
422/*
423 * Lets check how many CPUs we have.
424 */
425
Heiko Carstens255acee2006-02-17 13:52:46 -0800426static unsigned int
427__init smp_count_cpus(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428{
Heiko Carstens255acee2006-02-17 13:52:46 -0800429 unsigned int cpu, num_cpus;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 __u16 boot_cpu_addr;
431
432 /*
433 * cpu 0 is the boot cpu. See smp_prepare_boot_cpu.
434 */
435
436 boot_cpu_addr = S390_lowcore.cpu_data.cpu_addr;
437 current_thread_info()->cpu = 0;
438 num_cpus = 1;
Heiko Carstens255acee2006-02-17 13:52:46 -0800439 for (cpu = 0; cpu <= 65535; cpu++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 if ((__u16) cpu == boot_cpu_addr)
441 continue;
Heiko Carstens255acee2006-02-17 13:52:46 -0800442 __cpu_logical_map[1] = (__u16) cpu;
443 if (signal_processor(1, sigp_sense) ==
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 sigp_not_operational)
445 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 num_cpus++;
447 }
448
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 printk("Detected %d CPU's\n",(int) num_cpus);
450 printk("Boot cpu address %2X\n", boot_cpu_addr);
Heiko Carstens255acee2006-02-17 13:52:46 -0800451
452 return num_cpus;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453}
454
455/*
456 * Activate a secondary processor.
457 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458int __devinit start_secondary(void *cpuvoid)
459{
460 /* Setup the cpu */
461 cpu_init();
Nick Piggin5bfb5d62005-11-08 21:39:01 -0800462 preempt_disable();
Martin Schwidefskyd54853e2007-02-05 21:18:19 +0100463 /* Enable TOD clock interrupts on the secondary cpu. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 init_cpu_timer();
465#ifdef CONFIG_VIRT_TIMER
Martin Schwidefskyd54853e2007-02-05 21:18:19 +0100466 /* Enable cpu timer interrupts on the secondary cpu. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 init_cpu_vtimer();
468#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 /* Enable pfault pseudo page faults on this cpu. */
Heiko Carstens29b08d22006-12-04 15:40:40 +0100470 pfault_init();
471
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 /* Mark this cpu as online */
473 cpu_set(smp_processor_id(), cpu_online_map);
474 /* Switch on interrupts */
475 local_irq_enable();
476 /* Print info about this processor */
477 print_cpu_info(&S390_lowcore.cpu_data);
478 /* cpu_idle will call schedule for us */
479 cpu_idle();
480 return 0;
481}
482
483static void __init smp_create_idle(unsigned int cpu)
484{
485 struct task_struct *p;
486
487 /*
488 * don't care about the psw and regs settings since we'll never
489 * reschedule the forked task.
490 */
491 p = fork_idle(cpu);
492 if (IS_ERR(p))
493 panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
494 current_set[cpu] = p;
495}
496
497/* Reserving and releasing of CPUs */
498
499static DEFINE_SPINLOCK(smp_reserve_lock);
500static int smp_cpu_reserved[NR_CPUS];
501
502int
503smp_get_cpu(cpumask_t cpu_mask)
504{
505 unsigned long flags;
506 int cpu;
507
508 spin_lock_irqsave(&smp_reserve_lock, flags);
509 /* Try to find an already reserved cpu. */
510 for_each_cpu_mask(cpu, cpu_mask) {
511 if (smp_cpu_reserved[cpu] != 0) {
512 smp_cpu_reserved[cpu]++;
513 /* Found one. */
514 goto out;
515 }
516 }
517 /* Reserve a new cpu from cpu_mask. */
518 for_each_cpu_mask(cpu, cpu_mask) {
519 if (cpu_online(cpu)) {
520 smp_cpu_reserved[cpu]++;
521 goto out;
522 }
523 }
524 cpu = -ENODEV;
525out:
526 spin_unlock_irqrestore(&smp_reserve_lock, flags);
527 return cpu;
528}
529
530void
531smp_put_cpu(int cpu)
532{
533 unsigned long flags;
534
535 spin_lock_irqsave(&smp_reserve_lock, flags);
536 smp_cpu_reserved[cpu]--;
537 spin_unlock_irqrestore(&smp_reserve_lock, flags);
538}
539
Heiko Carstens4d284ca2007-02-05 21:18:53 +0100540static int
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541cpu_stopped(int cpu)
542{
543 __u32 status;
544
545 /* Check for stopped state */
546 if (signal_processor_ps(&status, 0, cpu, sigp_sense) == sigp_status_stored) {
547 if (status & 0x40)
548 return 1;
549 }
550 return 0;
551}
552
553/* Upping and downing of CPUs */
554
555int
556__cpu_up(unsigned int cpu)
557{
558 struct task_struct *idle;
559 struct _lowcore *cpu_lowcore;
560 struct stack_frame *sf;
561 sigp_ccode ccode;
562 int curr_cpu;
563
564 for (curr_cpu = 0; curr_cpu <= 65535; curr_cpu++) {
565 __cpu_logical_map[cpu] = (__u16) curr_cpu;
566 if (cpu_stopped(cpu))
567 break;
568 }
569
570 if (!cpu_stopped(cpu))
571 return -ENODEV;
572
573 ccode = signal_processor_p((__u32)(unsigned long)(lowcore_ptr[cpu]),
574 cpu, sigp_set_prefix);
575 if (ccode){
576 printk("sigp_set_prefix failed for cpu %d "
577 "with condition code %d\n",
578 (int) cpu, (int) ccode);
579 return -EIO;
580 }
581
582 idle = current_set[cpu];
583 cpu_lowcore = lowcore_ptr[cpu];
584 cpu_lowcore->kernel_stack = (unsigned long)
Al Viro30af7122006-01-12 01:05:50 -0800585 task_stack_page(idle) + (THREAD_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
587 - sizeof(struct pt_regs)
588 - sizeof(struct stack_frame));
589 memset(sf, 0, sizeof(struct stack_frame));
590 sf->gprs[9] = (unsigned long) sf;
591 cpu_lowcore->save_area[15] = (unsigned long) sf;
592 __ctl_store(cpu_lowcore->cregs_save_area[0], 0, 15);
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200593 asm volatile(
594 " stam 0,15,0(%0)"
595 : : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
597 cpu_lowcore->current_task = (unsigned long) idle;
598 cpu_lowcore->cpu_data.cpu_nr = cpu;
599 eieio();
Michael Ryan699ff132006-03-24 03:15:17 -0800600
601 while (signal_processor(cpu,sigp_restart) == sigp_busy)
602 udelay(10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603
604 while (!cpu_online(cpu))
605 cpu_relax();
606 return 0;
607}
608
Heiko Carstens255acee2006-02-17 13:52:46 -0800609static unsigned int __initdata additional_cpus;
Heiko Carstens37a33022006-02-17 13:52:47 -0800610static unsigned int __initdata possible_cpus;
Heiko Carstens255acee2006-02-17 13:52:46 -0800611
612void __init smp_setup_cpu_possible_map(void)
613{
Heiko Carstens54330452006-02-17 13:52:48 -0800614 unsigned int phy_cpus, pos_cpus, cpu;
Heiko Carstens255acee2006-02-17 13:52:46 -0800615
Heiko Carstens54330452006-02-17 13:52:48 -0800616 phy_cpus = smp_count_cpus();
617 pos_cpus = min(phy_cpus + additional_cpus, (unsigned int) NR_CPUS);
Heiko Carstens255acee2006-02-17 13:52:46 -0800618
Heiko Carstens37a33022006-02-17 13:52:47 -0800619 if (possible_cpus)
Heiko Carstens54330452006-02-17 13:52:48 -0800620 pos_cpus = min(possible_cpus, (unsigned int) NR_CPUS);
Heiko Carstens255acee2006-02-17 13:52:46 -0800621
Heiko Carstens54330452006-02-17 13:52:48 -0800622 for (cpu = 0; cpu < pos_cpus; cpu++)
Heiko Carstens255acee2006-02-17 13:52:46 -0800623 cpu_set(cpu, cpu_possible_map);
624
Heiko Carstens54330452006-02-17 13:52:48 -0800625 phy_cpus = min(phy_cpus, pos_cpus);
626
627 for (cpu = 0; cpu < phy_cpus; cpu++)
628 cpu_set(cpu, cpu_present_map);
Heiko Carstens255acee2006-02-17 13:52:46 -0800629}
630
631#ifdef CONFIG_HOTPLUG_CPU
632
633static int __init setup_additional_cpus(char *s)
634{
635 additional_cpus = simple_strtoul(s, NULL, 0);
636 return 0;
637}
638early_param("additional_cpus", setup_additional_cpus);
639
Heiko Carstens37a33022006-02-17 13:52:47 -0800640static int __init setup_possible_cpus(char *s)
641{
642 possible_cpus = simple_strtoul(s, NULL, 0);
643 return 0;
644}
645early_param("possible_cpus", setup_possible_cpus);
646
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647int
648__cpu_disable(void)
649{
650 unsigned long flags;
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200651 struct ec_creg_mask_parms cr_parms;
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700652 int cpu = smp_processor_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653
654 spin_lock_irqsave(&smp_reserve_lock, flags);
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700655 if (smp_cpu_reserved[cpu] != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 spin_unlock_irqrestore(&smp_reserve_lock, flags);
657 return -EBUSY;
658 }
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700659 cpu_clear(cpu, cpu_online_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 /* Disable pfault pseudo page faults on this cpu. */
Heiko Carstens29b08d22006-12-04 15:40:40 +0100662 pfault_fini();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200664 memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals));
665 memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200667 /* disable all external interrupts */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 cr_parms.orvals[0] = 0;
669 cr_parms.andvals[0] = ~(1<<15 | 1<<14 | 1<<13 | 1<<12 |
670 1<<11 | 1<<10 | 1<< 6 | 1<< 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 /* disable all I/O interrupts */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672 cr_parms.orvals[6] = 0;
673 cr_parms.andvals[6] = ~(1<<31 | 1<<30 | 1<<29 | 1<<28 |
674 1<<27 | 1<<26 | 1<<25 | 1<<24);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675 /* disable most machine checks */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 cr_parms.orvals[14] = 0;
677 cr_parms.andvals[14] = ~(1<<28 | 1<<27 | 1<<26 | 1<<25 | 1<<24);
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200678
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 smp_ctl_bit_callback(&cr_parms);
680
681 spin_unlock_irqrestore(&smp_reserve_lock, flags);
682 return 0;
683}
684
685void
686__cpu_die(unsigned int cpu)
687{
688 /* Wait until target cpu is down */
689 while (!smp_cpu_not_running(cpu))
690 cpu_relax();
691 printk("Processor %d spun down\n", cpu);
692}
693
694void
695cpu_die(void)
696{
697 idle_task_exit();
698 signal_processor(smp_processor_id(), sigp_stop);
699 BUG();
700 for(;;);
701}
702
Heiko Carstens255acee2006-02-17 13:52:46 -0800703#endif /* CONFIG_HOTPLUG_CPU */
704
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705/*
706 * Cycle through the processors and setup structures.
707 */
708
709void __init smp_prepare_cpus(unsigned int max_cpus)
710{
711 unsigned long stack;
712 unsigned int cpu;
713 int i;
714
Heiko Carstens99b2d8d2005-07-27 11:45:00 -0700715 /* request the 0x1201 emergency signal external interrupt */
716 if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
717 panic("Couldn't request external interrupt 0x1201");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 memset(lowcore_ptr,0,sizeof(lowcore_ptr));
719 /*
720 * Initialize prefix pages and stacks for all possible cpus
721 */
722 print_cpu_info(&S390_lowcore.cpu_data);
723
KAMEZAWA Hiroyuki97db7fb2006-03-31 02:30:26 -0800724 for_each_possible_cpu(i) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 lowcore_ptr[i] = (struct _lowcore *)
726 __get_free_pages(GFP_KERNEL|GFP_DMA,
727 sizeof(void*) == 8 ? 1 : 0);
728 stack = __get_free_pages(GFP_KERNEL,ASYNC_ORDER);
729 if (lowcore_ptr[i] == NULL || stack == 0ULL)
730 panic("smp_boot_cpus failed to allocate memory\n");
731
732 *(lowcore_ptr[i]) = S390_lowcore;
733 lowcore_ptr[i]->async_stack = stack + (ASYNC_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 stack = __get_free_pages(GFP_KERNEL,0);
735 if (stack == 0ULL)
736 panic("smp_boot_cpus failed to allocate memory\n");
737 lowcore_ptr[i]->panic_stack = stack + (PAGE_SIZE);
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -0800738#ifndef CONFIG_64BIT
Heiko Carstens77fa2242005-06-25 14:55:30 -0700739 if (MACHINE_HAS_IEEE) {
740 lowcore_ptr[i]->extended_save_area_addr =
741 (__u32) __get_free_pages(GFP_KERNEL,0);
742 if (lowcore_ptr[i]->extended_save_area_addr == 0)
743 panic("smp_boot_cpus failed to "
744 "allocate memory\n");
745 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746#endif
747 }
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -0800748#ifndef CONFIG_64BIT
Heiko Carstens77fa2242005-06-25 14:55:30 -0700749 if (MACHINE_HAS_IEEE)
750 ctl_set_bit(14, 29); /* enable extended save area */
751#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 set_prefix((u32)(unsigned long) lowcore_ptr[smp_processor_id()]);
753
KAMEZAWA Hiroyuki97db7fb2006-03-31 02:30:26 -0800754 for_each_possible_cpu(cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755 if (cpu != smp_processor_id())
756 smp_create_idle(cpu);
757}
758
759void __devinit smp_prepare_boot_cpu(void)
760{
761 BUG_ON(smp_processor_id() != 0);
762
763 cpu_set(0, cpu_online_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 S390_lowcore.percpu_offset = __per_cpu_offset[0];
765 current_set[0] = current;
766}
767
768void smp_cpus_done(unsigned int max_cpus)
769{
Heiko Carstens54330452006-02-17 13:52:48 -0800770 cpu_present_map = cpu_possible_map;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771}
772
773/*
774 * the frequency of the profiling timer can be changed
775 * by writing a multiplier value into /proc/profile.
776 *
777 * usually you want to run this on all CPUs ;)
778 */
779int setup_profiling_timer(unsigned int multiplier)
780{
781 return 0;
782}
783
784static DEFINE_PER_CPU(struct cpu, cpu_devices);
785
786static int __init topology_init(void)
787{
788 int cpu;
789 int ret;
790
KAMEZAWA Hiroyuki97db7fb2006-03-31 02:30:26 -0800791 for_each_possible_cpu(cpu) {
Heiko Carstens6721f772007-01-09 10:18:44 +0100792 struct cpu *c = &per_cpu(cpu_devices, cpu);
793
794 c->hotpluggable = 1;
795 ret = register_cpu(c, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796 if (ret)
797 printk(KERN_WARNING "topology_init: register_cpu %d "
798 "failed (%d)\n", cpu, ret);
799 }
800 return 0;
801}
802
803subsys_initcall(topology_init);
804
Heiko Carstens255acee2006-02-17 13:52:46 -0800805EXPORT_SYMBOL(cpu_online_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806EXPORT_SYMBOL(cpu_possible_map);
807EXPORT_SYMBOL(lowcore_ptr);
808EXPORT_SYMBOL(smp_ctl_set_bit);
809EXPORT_SYMBOL(smp_ctl_clear_bit);
810EXPORT_SYMBOL(smp_call_function);
811EXPORT_SYMBOL(smp_get_cpu);
812EXPORT_SYMBOL(smp_put_cpu);