blob: 15a016c1056392969bf84af7d4c1afe34aace480 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -04002 * SMP related functions
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
Heiko Carstensa53c8fa2012-07-20 11:15:04 +02004 * Copyright IBM Corp. 1999, 2012
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -04005 * Author(s): Denis Joseph Barrow,
6 * Martin Schwidefsky <schwidefsky@de.ibm.com>,
7 * Heiko Carstens <heiko.carstens@de.ibm.com>,
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 *
Heiko Carstens39ce0102007-04-27 16:02:00 +02009 * based on other smp stuff by
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net>
11 * (c) 1998 Ingo Molnar
12 *
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040013 * The code outside of smp.c uses logical cpu numbers, only smp.c does
14 * the translation of logical to physical cpu ids. All new code that
15 * operates on physical cpu numbers needs to go into smp.c.
Linus Torvalds1da177e2005-04-16 15:20:36 -070016 */
17
Martin Schwidefsky395d31d2008-12-25 13:39:50 +010018#define KMSG_COMPONENT "cpu"
19#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
20
Heiko Carstensf2308862011-01-05 12:48:08 +010021#include <linux/workqueue.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/module.h>
23#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/mm.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040025#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/spinlock.h>
27#include <linux/kernel_stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/interrupt.h>
Christian Borntraeger3324e602009-03-26 15:23:56 +010030#include <linux/irqflags.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070031#include <linux/cpu.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090032#include <linux/slab.h>
Michael Holzheu60a0c682011-10-30 15:16:40 +010033#include <linux/crash_dump.h>
Heiko Carstenscbb870c2010-02-26 22:37:43 +010034#include <asm/asm-offsets.h>
Heiko Carstens1e3cab22012-03-30 09:40:55 +020035#include <asm/switch_to.h>
36#include <asm/facility.h>
Michael Holzheu46b05d22007-02-21 10:55:21 +010037#include <asm/ipl.h>
Heiko Carstens2b67fc42007-02-05 21:16:47 +010038#include <asm/setup.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <asm/irq.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#include <asm/tlbflush.h>
Martin Schwidefsky27f6b412012-07-20 11:15:08 +020041#include <asm/vtimer.h>
Michael Holzheu411ed322007-04-27 16:01:49 +020042#include <asm/lowcore.h>
Heiko Carstens08d07962008-01-26 14:10:56 +010043#include <asm/sclp.h>
Martin Schwidefskyc742b312008-12-31 15:11:42 +010044#include <asm/vdso.h>
Michael Holzheu3ab121a2012-03-11 11:59:32 -040045#include <asm/debug.h>
Michael Holzheu4857d4b2012-03-11 11:59:34 -040046#include <asm/os_info.h>
Heiko Carstensa9ae32c2012-06-04 12:55:15 +020047#include <asm/sigp.h>
Heiko Carstensa8061702008-04-17 07:46:26 +020048#include "entry.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040050enum {
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040051 ec_schedule = 0,
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040052 ec_call_function_single,
53 ec_stop_cpu,
54};
Heiko Carstens08d07962008-01-26 14:10:56 +010055
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040056enum {
Heiko Carstens08d07962008-01-26 14:10:56 +010057 CPU_STATE_STANDBY,
58 CPU_STATE_CONFIGURED,
59};
60
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040061struct pcpu {
62 struct cpu cpu;
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040063 struct _lowcore *lowcore; /* lowcore page(s) for the cpu */
64 unsigned long async_stack; /* async stack for the cpu */
65 unsigned long panic_stack; /* panic stack for the cpu */
66 unsigned long ec_mask; /* bit mask for ec_xxx functions */
67 int state; /* physical cpu state */
Heiko Carstens50ab9a92012-09-04 17:36:16 +020068 int polarization; /* physical polarization */
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040069 u16 address; /* physical cpu address */
70};
71
72static u8 boot_cpu_type;
73static u16 boot_cpu_address;
74static struct pcpu pcpu_devices[NR_CPUS];
75
Heiko Carstens50ab9a92012-09-04 17:36:16 +020076/*
77 * The smp_cpu_state_mutex must be held when changing the state or polarization
78 * member of a pcpu data structure within the pcpu_devices arreay.
79 */
Heiko Carstensdbd70fb2008-04-17 07:46:12 +020080DEFINE_MUTEX(smp_cpu_state_mutex);
Heiko Carstens08d07962008-01-26 14:10:56 +010081
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040082/*
83 * Signal processor helper functions.
84 */
85static inline int __pcpu_sigp(u16 addr, u8 order, u32 parm, u32 *status)
Heiko Carstens5c0b9122009-09-11 10:29:05 +020086{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040087 register unsigned int reg1 asm ("1") = parm;
88 int cc;
Heiko Carstens5c0b9122009-09-11 10:29:05 +020089
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -040090 asm volatile(
91 " sigp %1,%2,0(%3)\n"
92 " ipm %0\n"
93 " srl %0,28\n"
94 : "=d" (cc), "+d" (reg1) : "d" (addr), "a" (order) : "cc");
95 if (status && cc == 1)
96 *status = reg1;
97 return cc;
Heiko Carstens5c0b9122009-09-11 10:29:05 +020098}
99
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400100static inline int __pcpu_sigp_relax(u16 addr, u8 order, u32 parm, u32 *status)
Heiko Carstensa93b8ec2010-02-26 22:37:35 +0100101{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400102 int cc;
103
104 while (1) {
Heiko Carstensc5e3acd2012-08-30 14:24:42 +0200105 cc = __pcpu_sigp(addr, order, parm, NULL);
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200106 if (cc != SIGP_CC_BUSY)
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400107 return cc;
108 cpu_relax();
109 }
110}
111
112static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm)
113{
114 int cc, retry;
115
116 for (retry = 0; ; retry++) {
Heiko Carstensc5e3acd2012-08-30 14:24:42 +0200117 cc = __pcpu_sigp(pcpu->address, order, parm, NULL);
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200118 if (cc != SIGP_CC_BUSY)
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400119 break;
120 if (retry >= 3)
121 udelay(10);
122 }
123 return cc;
124}
125
126static inline int pcpu_stopped(struct pcpu *pcpu)
127{
Heiko Carstens41459d32012-09-14 11:09:52 +0200128 u32 uninitialized_var(status);
Heiko Carstensc5e3acd2012-08-30 14:24:42 +0200129
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200130 if (__pcpu_sigp(pcpu->address, SIGP_SENSE,
Heiko Carstensc5e3acd2012-08-30 14:24:42 +0200131 0, &status) != SIGP_CC_STATUS_STORED)
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400132 return 0;
Heiko Carstensc5e3acd2012-08-30 14:24:42 +0200133 return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400134}
135
136static inline int pcpu_running(struct pcpu *pcpu)
137{
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200138 if (__pcpu_sigp(pcpu->address, SIGP_SENSE_RUNNING,
Heiko Carstensc5e3acd2012-08-30 14:24:42 +0200139 0, NULL) != SIGP_CC_STATUS_STORED)
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400140 return 1;
Heiko Carstens524b24a2012-06-04 12:11:41 +0200141 /* Status stored condition code is equivalent to cpu not running. */
142 return 0;
Heiko Carstensa93b8ec2010-02-26 22:37:35 +0100143}
144
Michael Holzheu1943f532011-10-30 15:16:38 +0100145/*
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400146 * Find struct pcpu by cpu address.
Michael Holzheu1943f532011-10-30 15:16:38 +0100147 */
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400148static struct pcpu *pcpu_find_address(const struct cpumask *mask, int address)
Michael Holzheu1943f532011-10-30 15:16:38 +0100149{
150 int cpu;
151
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400152 for_each_cpu(cpu, mask)
153 if (pcpu_devices[cpu].address == address)
154 return pcpu_devices + cpu;
155 return NULL;
156}
157
158static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit)
159{
160 int order;
161
162 set_bit(ec_bit, &pcpu->ec_mask);
163 order = pcpu_running(pcpu) ?
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200164 SIGP_EXTERNAL_CALL : SIGP_EMERGENCY_SIGNAL;
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400165 pcpu_sigp_retry(pcpu, order, 0);
166}
167
168static int __cpuinit pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
169{
170 struct _lowcore *lc;
171
172 if (pcpu != &pcpu_devices[0]) {
173 pcpu->lowcore = (struct _lowcore *)
174 __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
175 pcpu->async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
176 pcpu->panic_stack = __get_free_page(GFP_KERNEL);
177 if (!pcpu->lowcore || !pcpu->panic_stack || !pcpu->async_stack)
178 goto out;
Michael Holzheu1943f532011-10-30 15:16:38 +0100179 }
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400180 lc = pcpu->lowcore;
181 memcpy(lc, &S390_lowcore, 512);
182 memset((char *) lc + 512, 0, sizeof(*lc) - 512);
Martin Schwidefskydc7ee002013-04-24 10:20:43 +0200183 lc->async_stack = pcpu->async_stack + ASYNC_SIZE
184 - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
185 lc->panic_stack = pcpu->panic_stack + PAGE_SIZE
186 - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400187 lc->cpu_nr = cpu;
188#ifndef CONFIG_64BIT
189 if (MACHINE_HAS_IEEE) {
190 lc->extended_save_area_addr = get_zeroed_page(GFP_KERNEL);
191 if (!lc->extended_save_area_addr)
192 goto out;
193 }
194#else
195 if (vdso_alloc_per_cpu(lc))
196 goto out;
197#endif
198 lowcore_ptr[cpu] = lc;
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200199 pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, (u32)(unsigned long) lc);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400200 return 0;
201out:
202 if (pcpu != &pcpu_devices[0]) {
203 free_page(pcpu->panic_stack);
204 free_pages(pcpu->async_stack, ASYNC_ORDER);
205 free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
206 }
207 return -ENOMEM;
Michael Holzheu1943f532011-10-30 15:16:38 +0100208}
209
Heiko Carstens9d0f46a2012-05-09 16:27:35 +0200210#ifdef CONFIG_HOTPLUG_CPU
211
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400212static void pcpu_free_lowcore(struct pcpu *pcpu)
Heiko Carstens2c2df112010-02-26 22:37:34 +0100213{
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200214 pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400215 lowcore_ptr[pcpu - pcpu_devices] = NULL;
216#ifndef CONFIG_64BIT
217 if (MACHINE_HAS_IEEE) {
218 struct _lowcore *lc = pcpu->lowcore;
Heiko Carstens2c2df112010-02-26 22:37:34 +0100219
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400220 free_page((unsigned long) lc->extended_save_area_addr);
221 lc->extended_save_area_addr = 0;
222 }
223#else
224 vdso_free_per_cpu(pcpu->lowcore);
225#endif
226 if (pcpu != &pcpu_devices[0]) {
227 free_page(pcpu->panic_stack);
228 free_pages(pcpu->async_stack, ASYNC_ORDER);
229 free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
230 }
Heiko Carstens2c2df112010-02-26 22:37:34 +0100231}
232
Heiko Carstens9d0f46a2012-05-09 16:27:35 +0200233#endif /* CONFIG_HOTPLUG_CPU */
234
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400235static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
Martin Schwidefsky85ac7ca2011-12-27 11:27:22 +0100236{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400237 struct _lowcore *lc = pcpu->lowcore;
238
239 atomic_inc(&init_mm.context.attach_count);
240 lc->cpu_nr = cpu;
241 lc->percpu_offset = __per_cpu_offset[cpu];
242 lc->kernel_asce = S390_lowcore.kernel_asce;
243 lc->machine_flags = S390_lowcore.machine_flags;
244 lc->ftrace_func = S390_lowcore.ftrace_func;
245 lc->user_timer = lc->system_timer = lc->steal_timer = 0;
246 __ctl_store(lc->cregs_save_area, 0, 15);
247 save_access_regs((unsigned int *) lc->access_regs_save_area);
248 memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list,
249 MAX_FACILITY_BIT/8);
Martin Schwidefsky85ac7ca2011-12-27 11:27:22 +0100250}
251
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400252static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
253{
254 struct _lowcore *lc = pcpu->lowcore;
255 struct thread_info *ti = task_thread_info(tsk);
256
Martin Schwidefskydc7ee002013-04-24 10:20:43 +0200257 lc->kernel_stack = (unsigned long) task_stack_page(tsk)
258 + THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400259 lc->thread_info = (unsigned long) task_thread_info(tsk);
260 lc->current_task = (unsigned long) tsk;
261 lc->user_timer = ti->user_timer;
262 lc->system_timer = ti->system_timer;
263 lc->steal_timer = 0;
264}
265
266static void pcpu_start_fn(struct pcpu *pcpu, void (*func)(void *), void *data)
267{
268 struct _lowcore *lc = pcpu->lowcore;
269
270 lc->restart_stack = lc->kernel_stack;
271 lc->restart_fn = (unsigned long) func;
272 lc->restart_data = (unsigned long) data;
273 lc->restart_source = -1UL;
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200274 pcpu_sigp_retry(pcpu, SIGP_RESTART, 0);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400275}
276
277/*
278 * Call function via PSW restart on pcpu and stop the current cpu.
279 */
280static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *),
281 void *data, unsigned long stack)
282{
Michael Holzheu061da3d2012-05-24 14:38:26 +0200283 struct _lowcore *lc = lowcore_ptr[pcpu - pcpu_devices];
Heiko Carstensfbe76562012-06-05 09:59:52 +0200284 unsigned long source_cpu = stap();
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400285
286 __load_psw_mask(psw_kernel_bits);
Heiko Carstensfbe76562012-06-05 09:59:52 +0200287 if (pcpu->address == source_cpu)
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400288 func(data); /* should not return */
289 /* Stop target cpu (if func returns this stops the current cpu). */
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200290 pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400291 /* Restart func on the target cpu and stop the current cpu. */
Heiko Carstensfbe76562012-06-05 09:59:52 +0200292 mem_assign_absolute(lc->restart_stack, stack);
293 mem_assign_absolute(lc->restart_fn, (unsigned long) func);
294 mem_assign_absolute(lc->restart_data, (unsigned long) data);
295 mem_assign_absolute(lc->restart_source, source_cpu);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400296 asm volatile(
Heiko Carstenseb546192012-06-04 15:05:43 +0200297 "0: sigp 0,%0,%2 # sigp restart to target cpu\n"
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400298 " brc 2,0b # busy, try again\n"
Heiko Carstenseb546192012-06-04 15:05:43 +0200299 "1: sigp 0,%1,%3 # sigp stop to current cpu\n"
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400300 " brc 2,1b # busy, try again\n"
Heiko Carstensfbe76562012-06-05 09:59:52 +0200301 : : "d" (pcpu->address), "d" (source_cpu),
Heiko Carstenseb546192012-06-04 15:05:43 +0200302 "K" (SIGP_RESTART), "K" (SIGP_STOP)
303 : "0", "1", "cc");
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400304 for (;;) ;
305}
306
307/*
308 * Call function on an online CPU.
309 */
310void smp_call_online_cpu(void (*func)(void *), void *data)
311{
312 struct pcpu *pcpu;
313
314 /* Use the current cpu if it is online. */
315 pcpu = pcpu_find_address(cpu_online_mask, stap());
316 if (!pcpu)
317 /* Use the first online cpu. */
318 pcpu = pcpu_devices + cpumask_first(cpu_online_mask);
319 pcpu_delegate(pcpu, func, data, (unsigned long) restart_stack);
320}
321
322/*
323 * Call function on the ipl CPU.
324 */
325void smp_call_ipl_cpu(void (*func)(void *), void *data)
326{
Michael Holzheuc6da39f2012-03-13 11:25:08 -0400327 pcpu_delegate(&pcpu_devices[0], func, data,
328 pcpu_devices->panic_stack + PAGE_SIZE);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400329}
330
331int smp_find_processor_id(u16 address)
332{
333 int cpu;
334
335 for_each_present_cpu(cpu)
336 if (pcpu_devices[cpu].address == address)
337 return cpu;
338 return -1;
339}
340
341int smp_vcpu_scheduled(int cpu)
342{
343 return pcpu_running(pcpu_devices + cpu);
344}
345
346void smp_yield(void)
347{
348 if (MACHINE_HAS_DIAG44)
349 asm volatile("diag 0,0,0x44");
350}
351
352void smp_yield_cpu(int cpu)
353{
354 if (MACHINE_HAS_DIAG9C)
355 asm volatile("diag %0,0,0x9c"
356 : : "d" (pcpu_devices[cpu].address));
357 else if (MACHINE_HAS_DIAG44)
358 asm volatile("diag 0,0,0x44");
359}
360
361/*
362 * Send cpus emergency shutdown signal. This gives the cpus the
363 * opportunity to complete outstanding interrupts.
364 */
365void smp_emergency_stop(cpumask_t *cpumask)
366{
367 u64 end;
368 int cpu;
369
Heiko Carstens1aae0562013-01-30 09:49:40 +0100370 end = get_tod_clock() + (1000000UL << 12);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400371 for_each_cpu(cpu, cpumask) {
372 struct pcpu *pcpu = pcpu_devices + cpu;
373 set_bit(ec_stop_cpu, &pcpu->ec_mask);
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200374 while (__pcpu_sigp(pcpu->address, SIGP_EMERGENCY_SIGNAL,
375 0, NULL) == SIGP_CC_BUSY &&
Heiko Carstens1aae0562013-01-30 09:49:40 +0100376 get_tod_clock() < end)
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400377 cpu_relax();
378 }
Heiko Carstens1aae0562013-01-30 09:49:40 +0100379 while (get_tod_clock() < end) {
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400380 for_each_cpu(cpu, cpumask)
381 if (pcpu_stopped(pcpu_devices + cpu))
382 cpumask_clear_cpu(cpu, cpumask);
383 if (cpumask_empty(cpumask))
384 break;
385 cpu_relax();
386 }
387}
388
389/*
390 * Stop all cpus but the current one.
391 */
Heiko Carstens677d7622007-11-20 11:13:37 +0100392void smp_send_stop(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393{
Martin Schwidefsky85ac7ca2011-12-27 11:27:22 +0100394 cpumask_t cpumask;
395 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
Heiko Carstens677d7622007-11-20 11:13:37 +0100397 /* Disable all interrupts/machine checks */
Martin Schwidefskyb50511e2011-10-30 15:16:50 +0100398 __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
Christian Borntraeger3324e602009-03-26 15:23:56 +0100399 trace_hardirqs_off();
Heiko Carstens677d7622007-11-20 11:13:37 +0100400
Michael Holzheu3ab121a2012-03-11 11:59:32 -0400401 debug_set_critical();
Martin Schwidefsky85ac7ca2011-12-27 11:27:22 +0100402 cpumask_copy(&cpumask, cpu_online_mask);
403 cpumask_clear_cpu(smp_processor_id(), &cpumask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400405 if (oops_in_progress)
406 smp_emergency_stop(&cpumask);
Martin Schwidefsky85ac7ca2011-12-27 11:27:22 +0100407
408 /* stop all processors */
409 for_each_cpu(cpu, &cpumask) {
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400410 struct pcpu *pcpu = pcpu_devices + cpu;
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200411 pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400412 while (!pcpu_stopped(pcpu))
Heiko Carstensc6b5b842006-12-04 15:40:33 +0100413 cpu_relax();
414 }
415}
416
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417/*
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400418 * Stop the current cpu.
419 */
420void smp_stop_cpu(void)
421{
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200422 pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400423 for (;;) ;
424}
425
426/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 * This is the main routine where commands issued by other
428 * cpus are handled.
429 */
Heiko Carstens9acf73b2013-05-22 10:24:37 +0200430static void smp_handle_ext_call(void)
431{
432 unsigned long bits;
433
434 /* handle bit signal external calls */
435 bits = xchg(&pcpu_devices[smp_processor_id()].ec_mask, 0);
436 if (test_bit(ec_stop_cpu, &bits))
437 smp_stop_cpu();
438 if (test_bit(ec_schedule, &bits))
439 scheduler_ipi();
Heiko Carstens9acf73b2013-05-22 10:24:37 +0200440 if (test_bit(ec_call_function_single, &bits))
441 generic_smp_call_function_single_interrupt();
442}
443
Heiko Carstensfde15c32012-03-11 11:59:31 -0400444static void do_ext_call_interrupt(struct ext_code ext_code,
Martin Schwidefskyf6649a72010-10-25 16:10:38 +0200445 unsigned int param32, unsigned long param64)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446{
Heiko Carstens9acf73b2013-05-22 10:24:37 +0200447 inc_irq_stat(ext_code.code == 0x1202 ? IRQEXT_EXC : IRQEXT_EMS);
448 smp_handle_ext_call();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449}
450
Rusty Russell630cd042009-09-24 09:34:45 -0600451void arch_send_call_function_ipi_mask(const struct cpumask *mask)
Heiko Carstensca9fc752008-12-25 13:38:39 +0100452{
453 int cpu;
454
Rusty Russell630cd042009-09-24 09:34:45 -0600455 for_each_cpu(cpu, mask)
Heiko Carstensb6ed49e2013-05-21 15:34:56 +0200456 pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
Heiko Carstensca9fc752008-12-25 13:38:39 +0100457}
458
459void arch_send_call_function_single_ipi(int cpu)
460{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400461 pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
Heiko Carstensca9fc752008-12-25 13:38:39 +0100462}
463
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -0800464#ifndef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465/*
466 * this function sends a 'purge tlb' signal to another CPU.
467 */
Heiko Carstensa8061702008-04-17 07:46:26 +0200468static void smp_ptlb_callback(void *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469{
Martin Schwidefskyba8a9222007-10-22 12:52:44 +0200470 __tlb_flush_local();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471}
472
473void smp_ptlb_all(void)
474{
Jens Axboe15c8b6c2008-05-09 09:39:44 +0200475 on_each_cpu(smp_ptlb_callback, NULL, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476}
477EXPORT_SYMBOL(smp_ptlb_all);
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -0800478#endif /* ! CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479
480/*
481 * this function sends a 'reschedule' IPI to another CPU.
482 * it goes straight through and wastes no time serializing
483 * anything. Worst case is that we lose a reschedule ...
484 */
485void smp_send_reschedule(int cpu)
486{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400487 pcpu_ec_call(pcpu_devices + cpu, ec_schedule);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488}
489
490/*
491 * parameter area for the set/clear control bit callbacks
492 */
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200493struct ec_creg_mask_parms {
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400494 unsigned long orval;
495 unsigned long andval;
496 int cr;
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200497};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498
499/*
500 * callback for setting/clearing control bits
501 */
Heiko Carstens39ce0102007-04-27 16:02:00 +0200502static void smp_ctl_bit_callback(void *info)
503{
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200504 struct ec_creg_mask_parms *pp = info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 unsigned long cregs[16];
Heiko Carstens39ce0102007-04-27 16:02:00 +0200506
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200507 __ctl_store(cregs, 0, 15);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400508 cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval;
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200509 __ctl_load(cregs, 0, 15);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510}
511
512/*
513 * Set a bit in a control register of all cpus
514 */
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200515void smp_ctl_set_bit(int cr, int bit)
516{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400517 struct ec_creg_mask_parms parms = { 1UL << bit, -1UL, cr };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518
Jens Axboe15c8b6c2008-05-09 09:39:44 +0200519 on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520}
Heiko Carstens39ce0102007-04-27 16:02:00 +0200521EXPORT_SYMBOL(smp_ctl_set_bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522
523/*
524 * Clear a bit in a control register of all cpus
525 */
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200526void smp_ctl_clear_bit(int cr, int bit)
527{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400528 struct ec_creg_mask_parms parms = { 0, ~(1UL << bit), cr };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529
Jens Axboe15c8b6c2008-05-09 09:39:44 +0200530 on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531}
Heiko Carstens39ce0102007-04-27 16:02:00 +0200532EXPORT_SYMBOL(smp_ctl_clear_bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533
Michael Holzheu60a0c682011-10-30 15:16:40 +0100534#if defined(CONFIG_ZFCPDUMP) || defined(CONFIG_CRASH_DUMP)
Michael Holzheu411ed322007-04-27 16:01:49 +0200535
Heiko Carstensf64ca212010-02-26 22:37:32 +0100536struct save_area *zfcpdump_save_areas[NR_CPUS + 1];
Michael Holzheu411ed322007-04-27 16:01:49 +0200537EXPORT_SYMBOL_GPL(zfcpdump_save_areas);
538
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400539static void __init smp_get_save_area(int cpu, u16 address)
Heiko Carstens08d07962008-01-26 14:10:56 +0100540{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400541 void *lc = pcpu_devices[0].lowcore;
542 struct save_area *save_area;
Heiko Carstens08d07962008-01-26 14:10:56 +0100543
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400544 if (is_kdump_kernel())
545 return;
546 if (!OLDMEM_BASE && (address == boot_cpu_address ||
547 ipl_info.type != IPL_TYPE_FCP_DUMP))
548 return;
549 if (cpu >= NR_CPUS) {
550 pr_warning("CPU %i exceeds the maximum %i and is excluded "
551 "from the dump\n", cpu, NR_CPUS - 1);
552 return;
Heiko Carstens08d07962008-01-26 14:10:56 +0100553 }
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400554 save_area = kmalloc(sizeof(struct save_area), GFP_KERNEL);
555 if (!save_area)
556 panic("could not allocate memory for save area\n");
557 zfcpdump_save_areas[cpu] = save_area;
558#ifdef CONFIG_CRASH_DUMP
559 if (address == boot_cpu_address) {
560 /* Copy the registers of the boot cpu. */
561 copy_oldmem_page(1, (void *) save_area, sizeof(*save_area),
562 SAVE_AREA_BASE - PAGE_SIZE, 0);
563 return;
564 }
565#endif
566 /* Get the registers of a non-boot cpu. */
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200567 __pcpu_sigp_relax(address, SIGP_STOP_AND_STORE_STATUS, 0, NULL);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400568 memcpy_real(save_area, lc + SAVE_AREA_BASE, sizeof(*save_area));
569}
570
571int smp_store_status(int cpu)
572{
573 struct pcpu *pcpu;
574
575 pcpu = pcpu_devices + cpu;
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200576 if (__pcpu_sigp_relax(pcpu->address, SIGP_STOP_AND_STORE_STATUS,
577 0, NULL) != SIGP_CC_ORDER_CODE_ACCEPTED)
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400578 return -EIO;
Heiko Carstens08d07962008-01-26 14:10:56 +0100579 return 0;
580}
581
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400582#else /* CONFIG_ZFCPDUMP || CONFIG_CRASH_DUMP */
Heiko Carstens08d07962008-01-26 14:10:56 +0100583
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400584static inline void smp_get_save_area(int cpu, u16 address) { }
Heiko Carstens08d07962008-01-26 14:10:56 +0100585
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400586#endif /* CONFIG_ZFCPDUMP || CONFIG_CRASH_DUMP */
587
Heiko Carstens50ab9a92012-09-04 17:36:16 +0200588void smp_cpu_set_polarization(int cpu, int val)
589{
590 pcpu_devices[cpu].polarization = val;
591}
592
593int smp_cpu_get_polarization(int cpu)
594{
595 return pcpu_devices[cpu].polarization;
596}
597
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400598static struct sclp_cpu_info *smp_get_cpu_info(void)
Heiko Carstens08d07962008-01-26 14:10:56 +0100599{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400600 static int use_sigp_detection;
Heiko Carstens08d07962008-01-26 14:10:56 +0100601 struct sclp_cpu_info *info;
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400602 int address;
Heiko Carstens08d07962008-01-26 14:10:56 +0100603
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400604 info = kzalloc(sizeof(*info), GFP_KERNEL);
605 if (info && (use_sigp_detection || sclp_get_cpu_info(info))) {
606 use_sigp_detection = 1;
607 for (address = 0; address <= MAX_CPU_ADDRESS; address++) {
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200608 if (__pcpu_sigp_relax(address, SIGP_SENSE, 0, NULL) ==
609 SIGP_CC_NOT_OPERATIONAL)
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400610 continue;
611 info->cpu[info->configured].address = address;
612 info->configured++;
613 }
614 info->combined = info->configured;
Heiko Carstens08d07962008-01-26 14:10:56 +0100615 }
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400616 return info;
Heiko Carstens08d07962008-01-26 14:10:56 +0100617}
618
Heiko Carstenseba61972012-12-27 14:03:36 +0100619static int __cpuinit smp_add_present_cpu(int cpu);
Heiko Carstens08d07962008-01-26 14:10:56 +0100620
Heiko Carstenseba61972012-12-27 14:03:36 +0100621static int __cpuinit __smp_rescan_cpus(struct sclp_cpu_info *info,
622 int sysfs_add)
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400623{
624 struct pcpu *pcpu;
625 cpumask_t avail;
626 int cpu, nr, i;
627
628 nr = 0;
KOSAKI Motohiro0f1959f2011-05-23 10:24:36 +0200629 cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400630 cpu = cpumask_first(&avail);
631 for (i = 0; (i < info->combined) && (cpu < nr_cpu_ids); i++) {
632 if (info->has_cpu_type && info->cpu[i].type != boot_cpu_type)
633 continue;
634 if (pcpu_find_address(cpu_present_mask, info->cpu[i].address))
635 continue;
636 pcpu = pcpu_devices + cpu;
637 pcpu->address = info->cpu[i].address;
Martin Schwidefskya4eeea42013-05-15 16:53:07 +0200638 pcpu->state = (i >= info->configured) ?
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400639 CPU_STATE_STANDBY : CPU_STATE_CONFIGURED;
Heiko Carstens50ab9a92012-09-04 17:36:16 +0200640 smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400641 set_cpu_present(cpu, true);
642 if (sysfs_add && smp_add_present_cpu(cpu) != 0)
643 set_cpu_present(cpu, false);
644 else
645 nr++;
646 cpu = cpumask_next(cpu, &avail);
647 }
648 return nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649}
650
Heiko Carstens48483b32008-01-26 14:11:05 +0100651static void __init smp_detect_cpus(void)
652{
653 unsigned int cpu, c_cpus, s_cpus;
654 struct sclp_cpu_info *info;
Heiko Carstens48483b32008-01-26 14:11:05 +0100655
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400656 info = smp_get_cpu_info();
Heiko Carstens48483b32008-01-26 14:11:05 +0100657 if (!info)
658 panic("smp_detect_cpus failed to allocate memory\n");
Heiko Carstens48483b32008-01-26 14:11:05 +0100659 if (info->has_cpu_type) {
660 for (cpu = 0; cpu < info->combined; cpu++) {
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400661 if (info->cpu[cpu].address != boot_cpu_address)
662 continue;
663 /* The boot cpu dictates the cpu type. */
664 boot_cpu_type = info->cpu[cpu].type;
665 break;
Heiko Carstens48483b32008-01-26 14:11:05 +0100666 }
667 }
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400668 c_cpus = s_cpus = 0;
Heiko Carstens48483b32008-01-26 14:11:05 +0100669 for (cpu = 0; cpu < info->combined; cpu++) {
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400670 if (info->has_cpu_type && info->cpu[cpu].type != boot_cpu_type)
Heiko Carstens48483b32008-01-26 14:11:05 +0100671 continue;
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400672 if (cpu < info->configured) {
673 smp_get_save_area(c_cpus, info->cpu[cpu].address);
674 c_cpus++;
675 } else
Heiko Carstens48483b32008-01-26 14:11:05 +0100676 s_cpus++;
Heiko Carstens48483b32008-01-26 14:11:05 +0100677 }
Martin Schwidefsky395d31d2008-12-25 13:39:50 +0100678 pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
Martin Schwidefsky9d40d2e2008-01-26 14:11:31 +0100679 get_online_cpus();
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400680 __smp_rescan_cpus(info, 0);
Martin Schwidefsky9d40d2e2008-01-26 14:11:31 +0100681 put_online_cpus();
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400682 kfree(info);
Heiko Carstens48483b32008-01-26 14:11:05 +0100683}
684
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685/*
Heiko Carstens39ce0102007-04-27 16:02:00 +0200686 * Activate a secondary processor.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 */
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400688static void __cpuinit smp_start_secondary(void *cpuvoid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689{
Heiko Carstens1aae0562013-01-30 09:49:40 +0100690 S390_lowcore.last_update_clock = get_tod_clock();
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400691 S390_lowcore.restart_stack = (unsigned long) restart_stack;
692 S390_lowcore.restart_fn = (unsigned long) do_restart;
693 S390_lowcore.restart_data = 0;
694 S390_lowcore.restart_source = -1UL;
695 restore_access_regs(S390_lowcore.access_regs_save_area);
696 __ctl_load(S390_lowcore.cregs_save_area, 0, 15);
697 __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
Heiko Carstens39ce0102007-04-27 16:02:00 +0200698 cpu_init();
Nick Piggin5bfb5d62005-11-08 21:39:01 -0800699 preempt_disable();
Heiko Carstens39ce0102007-04-27 16:02:00 +0200700 init_cpu_timer();
Heiko Carstens39ce0102007-04-27 16:02:00 +0200701 init_cpu_vtimer();
Heiko Carstens29b08d22006-12-04 15:40:40 +0100702 pfault_init();
Manfred Spraule545a612008-09-07 16:57:22 +0200703 notify_cpu_starting(smp_processor_id());
KOSAKI Motohiro0f1959f2011-05-23 10:24:36 +0200704 set_cpu_online(smp_processor_id(), true);
Heiko Carstens93f3b2e2013-01-02 16:54:12 +0100705 inc_irq_stat(CPU_RST);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 local_irq_enable();
Thomas Gleixner52c00652013-03-21 22:49:57 +0100707 cpu_startup_entry(CPUHP_ONLINE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708}
709
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710/* Upping and downing of CPUs */
Thomas Gleixner8239c252012-04-20 13:05:42 +0000711int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400713 struct pcpu *pcpu;
714 int rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400716 pcpu = pcpu_devices + cpu;
717 if (pcpu->state != CPU_STATE_CONFIGURED)
Heiko Carstens08d07962008-01-26 14:10:56 +0100718 return -EIO;
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200719 if (pcpu_sigp_retry(pcpu, SIGP_INITIAL_CPU_RESET, 0) !=
720 SIGP_CC_ORDER_CODE_ACCEPTED)
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400721 return -EIO;
Thomas Gleixnere80e7812012-04-20 13:05:52 +0000722
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400723 rc = pcpu_alloc_lowcore(pcpu, cpu);
724 if (rc)
725 return rc;
726 pcpu_prepare_secondary(pcpu, cpu);
Thomas Gleixnere80e7812012-04-20 13:05:52 +0000727 pcpu_attach_task(pcpu, tidle);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400728 pcpu_start_fn(pcpu, smp_start_secondary, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729 while (!cpu_online(cpu))
730 cpu_relax();
731 return 0;
732}
733
Heiko Carstens37a33022006-02-17 13:52:47 -0800734static int __init setup_possible_cpus(char *s)
735{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400736 int max, cpu;
Heiko Carstens48483b32008-01-26 14:11:05 +0100737
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400738 if (kstrtoint(s, 0, &max) < 0)
739 return 0;
Heiko Carstens88e01282009-04-14 15:36:25 +0200740 init_cpu_possible(cpumask_of(0));
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400741 for (cpu = 1; cpu < max && cpu < nr_cpu_ids; cpu++)
Rusty Russelldef6cfb2009-03-26 15:25:00 +0100742 set_cpu_possible(cpu, true);
Heiko Carstens37a33022006-02-17 13:52:47 -0800743 return 0;
744}
745early_param("possible_cpus", setup_possible_cpus);
746
Heiko Carstens48483b32008-01-26 14:11:05 +0100747#ifdef CONFIG_HOTPLUG_CPU
748
Heiko Carstens39ce0102007-04-27 16:02:00 +0200749int __cpu_disable(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400751 unsigned long cregs[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752
Heiko Carstens9acf73b2013-05-22 10:24:37 +0200753 /* Handle possible pending IPIs */
754 smp_handle_ext_call();
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400755 set_cpu_online(smp_processor_id(), false);
756 /* Disable pseudo page faults on this cpu. */
Heiko Carstens29b08d22006-12-04 15:40:40 +0100757 pfault_fini();
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400758 /* Disable interrupt sources via control register. */
759 __ctl_store(cregs, 0, 15);
760 cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */
761 cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */
762 cregs[14] &= ~0x1f000000UL; /* disable most machine checks */
763 __ctl_load(cregs, 0, 15);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 return 0;
765}
766
Heiko Carstens39ce0102007-04-27 16:02:00 +0200767void __cpu_die(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400769 struct pcpu *pcpu;
770
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771 /* Wait until target cpu is down */
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400772 pcpu = pcpu_devices + cpu;
773 while (!pcpu_stopped(pcpu))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774 cpu_relax();
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400775 pcpu_free_lowcore(pcpu);
Martin Schwidefsky050eef32010-08-24 09:26:21 +0200776 atomic_dec(&init_mm.context.attach_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777}
778
Heiko Carstensb456d942011-05-23 10:24:33 +0200779void __noreturn cpu_die(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780{
781 idle_task_exit();
Heiko Carstensa9ae32c2012-06-04 12:55:15 +0200782 pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400783 for (;;) ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784}
785
Heiko Carstens255acee2006-02-17 13:52:46 -0800786#endif /* CONFIG_HOTPLUG_CPU */
787
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788void __init smp_prepare_cpus(unsigned int max_cpus)
789{
Heiko Carstens39ce0102007-04-27 16:02:00 +0200790 /* request the 0x1201 emergency signal external interrupt */
791 if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
792 panic("Couldn't request external interrupt 0x1201");
Martin Schwidefskyd98e19c2011-10-30 15:16:58 +0100793 /* request the 0x1202 external call external interrupt */
794 if (register_external_interrupt(0x1202, do_ext_call_interrupt) != 0)
795 panic("Couldn't request external interrupt 0x1202");
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400796 smp_detect_cpus();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797}
798
Heiko Carstensea1f4ee2007-05-31 17:38:05 +0200799void __init smp_prepare_boot_cpu(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400801 struct pcpu *pcpu = pcpu_devices;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400803 boot_cpu_address = stap();
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400804 pcpu->state = CPU_STATE_CONFIGURED;
805 pcpu->address = boot_cpu_address;
806 pcpu->lowcore = (struct _lowcore *)(unsigned long) store_prefix();
Martin Schwidefskydc7ee002013-04-24 10:20:43 +0200807 pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE
808 + STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);
809 pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE
810 + STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400811 S390_lowcore.percpu_offset = __per_cpu_offset[0];
Heiko Carstens50ab9a92012-09-04 17:36:16 +0200812 smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);
KOSAKI Motohiro0f1959f2011-05-23 10:24:36 +0200813 set_cpu_present(0, true);
814 set_cpu_online(0, true);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815}
816
Heiko Carstensea1f4ee2007-05-31 17:38:05 +0200817void __init smp_cpus_done(unsigned int max_cpus)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819}
820
Heiko Carstens02beacc2010-01-13 20:44:34 +0100821void __init smp_setup_processor_id(void)
822{
823 S390_lowcore.cpu_nr = 0;
Heiko Carstens02beacc2010-01-13 20:44:34 +0100824}
825
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826/*
827 * the frequency of the profiling timer can be changed
828 * by writing a multiplier value into /proc/profile.
829 *
830 * usually you want to run this on all CPUs ;)
831 */
832int setup_profiling_timer(unsigned int multiplier)
833{
Heiko Carstens39ce0102007-04-27 16:02:00 +0200834 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835}
836
Heiko Carstens08d07962008-01-26 14:10:56 +0100837#ifdef CONFIG_HOTPLUG_CPU
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800838static ssize_t cpu_configure_show(struct device *dev,
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400839 struct device_attribute *attr, char *buf)
Heiko Carstens08d07962008-01-26 14:10:56 +0100840{
841 ssize_t count;
842
843 mutex_lock(&smp_cpu_state_mutex);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400844 count = sprintf(buf, "%d\n", pcpu_devices[dev->id].state);
Heiko Carstens08d07962008-01-26 14:10:56 +0100845 mutex_unlock(&smp_cpu_state_mutex);
846 return count;
847}
848
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800849static ssize_t cpu_configure_store(struct device *dev,
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400850 struct device_attribute *attr,
851 const char *buf, size_t count)
Heiko Carstens08d07962008-01-26 14:10:56 +0100852{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400853 struct pcpu *pcpu;
854 int cpu, val, rc;
Heiko Carstens08d07962008-01-26 14:10:56 +0100855 char delim;
856
857 if (sscanf(buf, "%d %c", &val, &delim) != 1)
858 return -EINVAL;
859 if (val != 0 && val != 1)
860 return -EINVAL;
Martin Schwidefsky9d40d2e2008-01-26 14:11:31 +0100861 get_online_cpus();
Heiko Carstens0b18d312008-04-30 13:38:36 +0200862 mutex_lock(&smp_cpu_state_mutex);
Heiko Carstens08d07962008-01-26 14:10:56 +0100863 rc = -EBUSY;
Heiko Carstens2c2df112010-02-26 22:37:34 +0100864 /* disallow configuration changes of online cpus and cpu 0 */
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400865 cpu = dev->id;
Heiko Carstens2c2df112010-02-26 22:37:34 +0100866 if (cpu_online(cpu) || cpu == 0)
Heiko Carstens08d07962008-01-26 14:10:56 +0100867 goto out;
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400868 pcpu = pcpu_devices + cpu;
Heiko Carstens08d07962008-01-26 14:10:56 +0100869 rc = 0;
870 switch (val) {
871 case 0:
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400872 if (pcpu->state != CPU_STATE_CONFIGURED)
873 break;
874 rc = sclp_cpu_deconfigure(pcpu->address);
875 if (rc)
876 break;
877 pcpu->state = CPU_STATE_STANDBY;
Heiko Carstens50ab9a92012-09-04 17:36:16 +0200878 smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400879 topology_expect_change();
Heiko Carstens08d07962008-01-26 14:10:56 +0100880 break;
881 case 1:
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400882 if (pcpu->state != CPU_STATE_STANDBY)
883 break;
884 rc = sclp_cpu_configure(pcpu->address);
885 if (rc)
886 break;
887 pcpu->state = CPU_STATE_CONFIGURED;
Heiko Carstens50ab9a92012-09-04 17:36:16 +0200888 smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400889 topology_expect_change();
Heiko Carstens08d07962008-01-26 14:10:56 +0100890 break;
891 default:
892 break;
893 }
894out:
Heiko Carstens08d07962008-01-26 14:10:56 +0100895 mutex_unlock(&smp_cpu_state_mutex);
Heiko Carstens0b18d312008-04-30 13:38:36 +0200896 put_online_cpus();
Heiko Carstens08d07962008-01-26 14:10:56 +0100897 return rc ? rc : count;
898}
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800899static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
Heiko Carstens08d07962008-01-26 14:10:56 +0100900#endif /* CONFIG_HOTPLUG_CPU */
901
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800902static ssize_t show_cpu_address(struct device *dev,
903 struct device_attribute *attr, char *buf)
Heiko Carstens08d07962008-01-26 14:10:56 +0100904{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400905 return sprintf(buf, "%d\n", pcpu_devices[dev->id].address);
Heiko Carstens08d07962008-01-26 14:10:56 +0100906}
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800907static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
Heiko Carstens08d07962008-01-26 14:10:56 +0100908
Heiko Carstens08d07962008-01-26 14:10:56 +0100909static struct attribute *cpu_common_attrs[] = {
910#ifdef CONFIG_HOTPLUG_CPU
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800911 &dev_attr_configure.attr,
Heiko Carstens08d07962008-01-26 14:10:56 +0100912#endif
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800913 &dev_attr_address.attr,
Heiko Carstens08d07962008-01-26 14:10:56 +0100914 NULL,
915};
916
917static struct attribute_group cpu_common_attr_group = {
918 .attrs = cpu_common_attrs,
919};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800921static ssize_t show_idle_count(struct device *dev,
922 struct device_attribute *attr, char *buf)
Heiko Carstensfae8b222007-10-22 12:52:39 +0200923{
Martin Schwidefsky4c1051e2012-03-11 11:59:27 -0400924 struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id);
Heiko Carstensfae8b222007-10-22 12:52:39 +0200925 unsigned long long idle_count;
Martin Schwidefskye98bbaa2009-06-22 12:08:20 +0200926 unsigned int sequence;
Heiko Carstensfae8b222007-10-22 12:52:39 +0200927
Martin Schwidefsky4c1051e2012-03-11 11:59:27 -0400928 do {
929 sequence = ACCESS_ONCE(idle->sequence);
930 idle_count = ACCESS_ONCE(idle->idle_count);
Martin Schwidefsky27f6b412012-07-20 11:15:08 +0200931 if (ACCESS_ONCE(idle->clock_idle_enter))
Martin Schwidefsky4c1051e2012-03-11 11:59:27 -0400932 idle_count++;
933 } while ((sequence & 1) || (idle->sequence != sequence));
Heiko Carstensfae8b222007-10-22 12:52:39 +0200934 return sprintf(buf, "%llu\n", idle_count);
935}
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800936static DEVICE_ATTR(idle_count, 0444, show_idle_count, NULL);
Heiko Carstensfae8b222007-10-22 12:52:39 +0200937
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800938static ssize_t show_idle_time(struct device *dev,
939 struct device_attribute *attr, char *buf)
Heiko Carstensfae8b222007-10-22 12:52:39 +0200940{
Martin Schwidefsky4c1051e2012-03-11 11:59:27 -0400941 struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id);
942 unsigned long long now, idle_time, idle_enter, idle_exit;
Martin Schwidefskye98bbaa2009-06-22 12:08:20 +0200943 unsigned int sequence;
Heiko Carstensfae8b222007-10-22 12:52:39 +0200944
Martin Schwidefsky4c1051e2012-03-11 11:59:27 -0400945 do {
Heiko Carstens1aae0562013-01-30 09:49:40 +0100946 now = get_tod_clock();
Martin Schwidefsky4c1051e2012-03-11 11:59:27 -0400947 sequence = ACCESS_ONCE(idle->sequence);
948 idle_time = ACCESS_ONCE(idle->idle_time);
Martin Schwidefsky27f6b412012-07-20 11:15:08 +0200949 idle_enter = ACCESS_ONCE(idle->clock_idle_enter);
950 idle_exit = ACCESS_ONCE(idle->clock_idle_exit);
Martin Schwidefsky4c1051e2012-03-11 11:59:27 -0400951 } while ((sequence & 1) || (idle->sequence != sequence));
952 idle_time += idle_enter ? ((idle_exit ? : now) - idle_enter) : 0;
Martin Schwidefsky6f430922008-12-31 15:11:40 +0100953 return sprintf(buf, "%llu\n", idle_time >> 12);
Heiko Carstensfae8b222007-10-22 12:52:39 +0200954}
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800955static DEVICE_ATTR(idle_time_us, 0444, show_idle_time, NULL);
Heiko Carstensfae8b222007-10-22 12:52:39 +0200956
Heiko Carstens08d07962008-01-26 14:10:56 +0100957static struct attribute *cpu_online_attrs[] = {
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800958 &dev_attr_idle_count.attr,
959 &dev_attr_idle_time_us.attr,
Heiko Carstensfae8b222007-10-22 12:52:39 +0200960 NULL,
961};
962
Heiko Carstens08d07962008-01-26 14:10:56 +0100963static struct attribute_group cpu_online_attr_group = {
964 .attrs = cpu_online_attrs,
Heiko Carstensfae8b222007-10-22 12:52:39 +0200965};
966
Heiko Carstens2fc2d1e2007-04-27 16:01:56 +0200967static int __cpuinit smp_cpu_notify(struct notifier_block *self,
968 unsigned long action, void *hcpu)
969{
970 unsigned int cpu = (unsigned int)(long)hcpu;
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400971 struct cpu *c = &pcpu_devices[cpu].cpu;
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800972 struct device *s = &c->dev;
Akinobu Mitad882ba62010-05-26 14:43:34 -0700973 int err = 0;
Heiko Carstens2fc2d1e2007-04-27 16:01:56 +0200974
Heiko Carstens1c725922012-08-27 15:43:49 +0200975 switch (action & ~CPU_TASKS_FROZEN) {
Heiko Carstens2fc2d1e2007-04-27 16:01:56 +0200976 case CPU_ONLINE:
Akinobu Mitad882ba62010-05-26 14:43:34 -0700977 err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
Heiko Carstens2fc2d1e2007-04-27 16:01:56 +0200978 break;
979 case CPU_DEAD:
Heiko Carstens08d07962008-01-26 14:10:56 +0100980 sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
Heiko Carstens2fc2d1e2007-04-27 16:01:56 +0200981 break;
982 }
Akinobu Mitad882ba62010-05-26 14:43:34 -0700983 return notifier_from_errno(err);
Heiko Carstens2fc2d1e2007-04-27 16:01:56 +0200984}
985
Heiko Carstenseba61972012-12-27 14:03:36 +0100986static int __cpuinit smp_add_present_cpu(int cpu)
Heiko Carstens08d07962008-01-26 14:10:56 +0100987{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -0400988 struct cpu *c = &pcpu_devices[cpu].cpu;
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800989 struct device *s = &c->dev;
Heiko Carstens08d07962008-01-26 14:10:56 +0100990 int rc;
991
992 c->hotpluggable = 1;
993 rc = register_cpu(c, cpu);
994 if (rc)
995 goto out;
996 rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
997 if (rc)
998 goto out_cpu;
Heiko Carstens83a24e32011-12-27 11:27:09 +0100999 if (cpu_online(cpu)) {
1000 rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
1001 if (rc)
1002 goto out_online;
1003 }
1004 rc = topology_cpu_init(c);
1005 if (rc)
1006 goto out_topology;
1007 return 0;
1008
1009out_topology:
1010 if (cpu_online(cpu))
1011 sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
1012out_online:
Heiko Carstens08d07962008-01-26 14:10:56 +01001013 sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
1014out_cpu:
1015#ifdef CONFIG_HOTPLUG_CPU
1016 unregister_cpu(c);
1017#endif
1018out:
1019 return rc;
1020}
1021
1022#ifdef CONFIG_HOTPLUG_CPU
Heiko Carstens1e489512008-04-30 13:38:37 +02001023
Heiko Carstens67060d92008-05-30 10:03:27 +02001024int __ref smp_rescan_cpus(void)
Heiko Carstens08d07962008-01-26 14:10:56 +01001025{
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -04001026 struct sclp_cpu_info *info;
1027 int nr;
Heiko Carstens08d07962008-01-26 14:10:56 +01001028
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -04001029 info = smp_get_cpu_info();
1030 if (!info)
1031 return -ENOMEM;
Martin Schwidefsky9d40d2e2008-01-26 14:11:31 +01001032 get_online_cpus();
Heiko Carstens0b18d312008-04-30 13:38:36 +02001033 mutex_lock(&smp_cpu_state_mutex);
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -04001034 nr = __smp_rescan_cpus(info, 1);
Heiko Carstens08d07962008-01-26 14:10:56 +01001035 mutex_unlock(&smp_cpu_state_mutex);
Heiko Carstens0b18d312008-04-30 13:38:36 +02001036 put_online_cpus();
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -04001037 kfree(info);
1038 if (nr)
Heiko Carstensc10fde02008-04-17 07:46:13 +02001039 topology_schedule_update();
Martin Schwidefsky8b646bd2012-03-11 11:59:26 -04001040 return 0;
Heiko Carstens1e489512008-04-30 13:38:37 +02001041}
1042
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001043static ssize_t __ref rescan_store(struct device *dev,
1044 struct device_attribute *attr,
Andi Kleenc9be0a32010-01-05 12:47:58 +01001045 const char *buf,
Heiko Carstens1e489512008-04-30 13:38:37 +02001046 size_t count)
1047{
1048 int rc;
1049
1050 rc = smp_rescan_cpus();
Heiko Carstens08d07962008-01-26 14:10:56 +01001051 return rc ? rc : count;
1052}
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001053static DEVICE_ATTR(rescan, 0200, NULL, rescan_store);
Heiko Carstens08d07962008-01-26 14:10:56 +01001054#endif /* CONFIG_HOTPLUG_CPU */
1055
Heiko Carstens83a24e32011-12-27 11:27:09 +01001056static int __init s390_smp_init(void)
Heiko Carstensc10fde02008-04-17 07:46:13 +02001057{
Heiko Carstens83a24e32011-12-27 11:27:09 +01001058 int cpu, rc;
Heiko Carstens2fc2d1e2007-04-27 16:01:56 +02001059
Heiko Carstens7755d6b2012-08-27 15:14:47 +02001060 hotcpu_notifier(smp_cpu_notify, 0);
Heiko Carstens08d07962008-01-26 14:10:56 +01001061#ifdef CONFIG_HOTPLUG_CPU
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001062 rc = device_create_file(cpu_subsys.dev_root, &dev_attr_rescan);
Heiko Carstens08d07962008-01-26 14:10:56 +01001063 if (rc)
1064 return rc;
1065#endif
1066 for_each_present_cpu(cpu) {
1067 rc = smp_add_present_cpu(cpu);
Heiko Carstensfae8b222007-10-22 12:52:39 +02001068 if (rc)
1069 return rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070 }
1071 return 0;
1072}
Heiko Carstens83a24e32011-12-27 11:27:09 +01001073subsys_initcall(s390_smp_init);