blob: 70dd8e5c68897a33c2fba8f3106663e034d62157 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86-64/kernel/process.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 *
6 * Pentium III FXSR, SSE support
7 * Gareth Hughes <gareth@valinux.com>, May 2000
8 *
9 * X86-64 port
10 * Andi Kleen.
Ashok Raj76e4f662005-06-25 14:55:00 -070011 *
12 * CPU hotplug support - ashok.raj@intel.com
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * $Id: process.c,v 1.38 2002/01/15 10:08:03 ak Exp $
14 */
15
16/*
17 * This file handles the architecture-dependent parts of process handling..
18 */
19
20#include <stdarg.h>
21
Ashok Raj76e4f662005-06-25 14:55:00 -070022#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/errno.h>
24#include <linux/sched.h>
25#include <linux/kernel.h>
26#include <linux/mm.h>
27#include <linux/elfcore.h>
28#include <linux/smp.h>
29#include <linux/slab.h>
30#include <linux/user.h>
31#include <linux/module.h>
32#include <linux/a.out.h>
33#include <linux/interrupt.h>
34#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <linux/ptrace.h>
36#include <linux/utsname.h>
37#include <linux/random.h>
Andi Kleen95833c82006-01-11 22:44:36 +010038#include <linux/notifier.h>
bibo maoc6fd91f2006-03-26 01:38:20 -080039#include <linux/kprobes.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
41#include <asm/uaccess.h>
42#include <asm/pgtable.h>
43#include <asm/system.h>
44#include <asm/io.h>
45#include <asm/processor.h>
46#include <asm/i387.h>
47#include <asm/mmu_context.h>
48#include <asm/pda.h>
49#include <asm/prctl.h>
50#include <asm/kdebug.h>
51#include <asm/desc.h>
52#include <asm/proto.h>
53#include <asm/ia32.h>
Andi Kleen95833c82006-01-11 22:44:36 +010054#include <asm/idle.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
56asmlinkage extern void ret_from_fork(void);
57
58unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
59
Linus Torvalds1da177e2005-04-16 15:20:36 -070060unsigned long boot_option_idle_override = 0;
61EXPORT_SYMBOL(boot_option_idle_override);
62
63/*
64 * Powermanagement idle function, if any..
65 */
66void (*pm_idle)(void);
67static DEFINE_PER_CPU(unsigned int, cpu_idle_state);
68
Alan Sterne041c682006-03-27 01:16:30 -080069static ATOMIC_NOTIFIER_HEAD(idle_notifier);
Andi Kleen95833c82006-01-11 22:44:36 +010070
71void idle_notifier_register(struct notifier_block *n)
72{
Alan Sterne041c682006-03-27 01:16:30 -080073 atomic_notifier_chain_register(&idle_notifier, n);
Andi Kleen95833c82006-01-11 22:44:36 +010074}
75EXPORT_SYMBOL_GPL(idle_notifier_register);
76
77void idle_notifier_unregister(struct notifier_block *n)
78{
Alan Sterne041c682006-03-27 01:16:30 -080079 atomic_notifier_chain_unregister(&idle_notifier, n);
Andi Kleen95833c82006-01-11 22:44:36 +010080}
81EXPORT_SYMBOL(idle_notifier_unregister);
82
83enum idle_state { CPU_IDLE, CPU_NOT_IDLE };
84static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE;
85
86void enter_idle(void)
87{
88 __get_cpu_var(idle_state) = CPU_IDLE;
Alan Sterne041c682006-03-27 01:16:30 -080089 atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
Andi Kleen95833c82006-01-11 22:44:36 +010090}
91
92static void __exit_idle(void)
93{
94 __get_cpu_var(idle_state) = CPU_NOT_IDLE;
Alan Sterne041c682006-03-27 01:16:30 -080095 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
Andi Kleen95833c82006-01-11 22:44:36 +010096}
97
98/* Called from interrupts to signify idle end */
99void exit_idle(void)
100{
101 if (current->pid | read_pda(irqcount))
102 return;
103 __exit_idle();
104}
105
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106/*
107 * We use this if we don't have any better
108 * idle routine..
109 */
Adrian Bunkcdb04522006-03-24 03:15:57 -0800110static void default_idle(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111{
Nick Piggin64c7c8f2005-11-08 21:39:04 -0800112 local_irq_enable();
113
Andi Kleen2d52ede2006-01-11 22:42:42 +0100114 clear_thread_flag(TIF_POLLING_NRFLAG);
115 smp_mb__after_clear_bit();
116 while (!need_resched()) {
117 local_irq_disable();
118 if (!need_resched())
119 safe_halt();
120 else
121 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 }
Andi Kleen2d52ede2006-01-11 22:42:42 +0100123 set_thread_flag(TIF_POLLING_NRFLAG);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124}
125
126/*
127 * On SMP it's slightly faster (but much more power-consuming!)
128 * to poll the ->need_resched flag instead of waiting for the
129 * cross-CPU IPI to arrive. Use this option with caution.
130 */
131static void poll_idle (void)
132{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133 local_irq_enable();
134
Nick Piggin64c7c8f2005-11-08 21:39:04 -0800135 asm volatile(
136 "2:"
137 "testl %0,%1;"
138 "rep; nop;"
139 "je 2b;"
140 : :
141 "i" (_TIF_NEED_RESCHED),
142 "m" (current_thread_info()->flags));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143}
144
145void cpu_idle_wait(void)
146{
147 unsigned int cpu, this_cpu = get_cpu();
148 cpumask_t map;
149
150 set_cpus_allowed(current, cpumask_of_cpu(this_cpu));
151 put_cpu();
152
153 cpus_clear(map);
154 for_each_online_cpu(cpu) {
155 per_cpu(cpu_idle_state, cpu) = 1;
156 cpu_set(cpu, map);
157 }
158
159 __get_cpu_var(cpu_idle_state) = 0;
160
161 wmb();
162 do {
163 ssleep(1);
164 for_each_online_cpu(cpu) {
Andi Kleena88cde12005-11-05 17:25:54 +0100165 if (cpu_isset(cpu, map) &&
166 !per_cpu(cpu_idle_state, cpu))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 cpu_clear(cpu, map);
168 }
169 cpus_and(map, map, cpu_online_map);
170 } while (!cpus_empty(map));
171}
172EXPORT_SYMBOL_GPL(cpu_idle_wait);
173
Ashok Raj76e4f662005-06-25 14:55:00 -0700174#ifdef CONFIG_HOTPLUG_CPU
175DECLARE_PER_CPU(int, cpu_state);
176
177#include <asm/nmi.h>
Shaohua Li1fa744e2006-01-06 00:12:20 -0800178/* We halt the CPU with physical CPU hotplug */
Ashok Raj76e4f662005-06-25 14:55:00 -0700179static inline void play_dead(void)
180{
181 idle_task_exit();
182 wbinvd();
183 mb();
184 /* Ack it */
185 __get_cpu_var(cpu_state) = CPU_DEAD;
186
Shaohua Li1fa744e2006-01-06 00:12:20 -0800187 local_irq_disable();
Ashok Raj76e4f662005-06-25 14:55:00 -0700188 while (1)
Shaohua Li1fa744e2006-01-06 00:12:20 -0800189 halt();
Ashok Raj76e4f662005-06-25 14:55:00 -0700190}
191#else
192static inline void play_dead(void)
193{
194 BUG();
195}
196#endif /* CONFIG_HOTPLUG_CPU */
197
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198/*
199 * The idle thread. There's no useful work to be
200 * done, so just try to conserve power and have a
201 * low exit latency (ie sit in a loop waiting for
202 * somebody to say that they'd like to reschedule)
203 */
204void cpu_idle (void)
205{
Nick Piggin64c7c8f2005-11-08 21:39:04 -0800206 set_thread_flag(TIF_POLLING_NRFLAG);
207
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208 /* endless idle loop with no priority at all */
209 while (1) {
210 while (!need_resched()) {
211 void (*idle)(void);
212
213 if (__get_cpu_var(cpu_idle_state))
214 __get_cpu_var(cpu_idle_state) = 0;
215
216 rmb();
217 idle = pm_idle;
218 if (!idle)
219 idle = default_idle;
Ashok Raj76e4f662005-06-25 14:55:00 -0700220 if (cpu_is_offline(smp_processor_id()))
221 play_dead();
Andi Kleen95833c82006-01-11 22:44:36 +0100222 enter_idle();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 idle();
Andi Kleen95833c82006-01-11 22:44:36 +0100224 __exit_idle();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 }
226
Nick Piggin5bfb5d62005-11-08 21:39:01 -0800227 preempt_enable_no_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 schedule();
Nick Piggin5bfb5d62005-11-08 21:39:01 -0800229 preempt_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 }
231}
232
233/*
234 * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
235 * which can obviate IPI to trigger checking of need_resched.
236 * We execute MONITOR against need_resched and enter optimized wait state
237 * through MWAIT. Whenever someone changes need_resched, we would be woken
238 * up from MWAIT (without an IPI).
239 */
240static void mwait_idle(void)
241{
242 local_irq_enable();
243
Nick Piggin64c7c8f2005-11-08 21:39:04 -0800244 while (!need_resched()) {
245 __monitor((void *)&current_thread_info()->flags, 0, 0);
246 smp_mb();
247 if (need_resched())
248 break;
249 __mwait(0, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 }
251}
252
Ashok Raje6982c62005-06-25 14:54:58 -0700253void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254{
255 static int printed;
256 if (cpu_has(c, X86_FEATURE_MWAIT)) {
257 /*
258 * Skip, if setup has overridden idle.
259 * One CPU supports mwait => All CPUs supports mwait
260 */
261 if (!pm_idle) {
262 if (!printed) {
263 printk("using mwait in idle threads.\n");
264 printed = 1;
265 }
266 pm_idle = mwait_idle;
267 }
268 }
269}
270
271static int __init idle_setup (char *str)
272{
273 if (!strncmp(str, "poll", 4)) {
274 printk("using polling idle threads.\n");
275 pm_idle = poll_idle;
276 }
277
278 boot_option_idle_override = 1;
279 return 1;
280}
281
282__setup("idle=", idle_setup);
283
284/* Prints also some state that isn't saved in the pt_regs */
285void __show_regs(struct pt_regs * regs)
286{
287 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
288 unsigned int fsindex,gsindex;
289 unsigned int ds,cs,es;
290
291 printk("\n");
292 print_modules();
Andi Kleen9acf23c2005-09-12 18:49:24 +0200293 printk("Pid: %d, comm: %.20s %s %s %.*s\n",
294 current->pid, current->comm, print_tainted(),
295 system_utsname.release,
296 (int)strcspn(system_utsname.version, " "),
297 system_utsname.version);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->rip);
299 printk_address(regs->rip);
Andi Kleena88cde12005-11-05 17:25:54 +0100300 printk("\nRSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss, regs->rsp,
301 regs->eflags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 printk("RAX: %016lx RBX: %016lx RCX: %016lx\n",
303 regs->rax, regs->rbx, regs->rcx);
304 printk("RDX: %016lx RSI: %016lx RDI: %016lx\n",
305 regs->rdx, regs->rsi, regs->rdi);
306 printk("RBP: %016lx R08: %016lx R09: %016lx\n",
307 regs->rbp, regs->r8, regs->r9);
308 printk("R10: %016lx R11: %016lx R12: %016lx\n",
309 regs->r10, regs->r11, regs->r12);
310 printk("R13: %016lx R14: %016lx R15: %016lx\n",
311 regs->r13, regs->r14, regs->r15);
312
313 asm("movl %%ds,%0" : "=r" (ds));
314 asm("movl %%cs,%0" : "=r" (cs));
315 asm("movl %%es,%0" : "=r" (es));
316 asm("movl %%fs,%0" : "=r" (fsindex));
317 asm("movl %%gs,%0" : "=r" (gsindex));
318
319 rdmsrl(MSR_FS_BASE, fs);
320 rdmsrl(MSR_GS_BASE, gs);
321 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
322
323 asm("movq %%cr0, %0": "=r" (cr0));
324 asm("movq %%cr2, %0": "=r" (cr2));
325 asm("movq %%cr3, %0": "=r" (cr3));
326 asm("movq %%cr4, %0": "=r" (cr4));
327
328 printk("FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
329 fs,fsindex,gs,gsindex,shadowgs);
330 printk("CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0);
331 printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4);
332}
333
334void show_regs(struct pt_regs *regs)
335{
Zwane Mwaikamboc078d322005-09-06 15:16:16 -0700336 printk("CPU %d:", smp_processor_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337 __show_regs(regs);
338 show_trace(&regs->rsp);
339}
340
341/*
342 * Free current thread data structures etc..
343 */
344void exit_thread(void)
345{
346 struct task_struct *me = current;
347 struct thread_struct *t = &me->thread;
Rusty Lynch73649da2005-06-23 00:09:23 -0700348
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 if (me->thread.io_bitmap_ptr) {
350 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
351
352 kfree(t->io_bitmap_ptr);
353 t->io_bitmap_ptr = NULL;
354 /*
355 * Careful, clear this in the TSS too:
356 */
357 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
358 t->io_bitmap_max = 0;
359 put_cpu();
360 }
361}
362
363void flush_thread(void)
364{
365 struct task_struct *tsk = current;
366 struct thread_info *t = current_thread_info();
367
368 if (t->flags & _TIF_ABI_PENDING)
369 t->flags ^= (_TIF_ABI_PENDING | _TIF_IA32);
370
371 tsk->thread.debugreg0 = 0;
372 tsk->thread.debugreg1 = 0;
373 tsk->thread.debugreg2 = 0;
374 tsk->thread.debugreg3 = 0;
375 tsk->thread.debugreg6 = 0;
376 tsk->thread.debugreg7 = 0;
377 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
378 /*
379 * Forget coprocessor state..
380 */
381 clear_fpu(tsk);
382 clear_used_math();
383}
384
385void release_thread(struct task_struct *dead_task)
386{
387 if (dead_task->mm) {
388 if (dead_task->mm->context.size) {
389 printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
390 dead_task->comm,
391 dead_task->mm->context.ldt,
392 dead_task->mm->context.size);
393 BUG();
394 }
395 }
396}
397
398static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
399{
400 struct user_desc ud = {
401 .base_addr = addr,
402 .limit = 0xfffff,
403 .seg_32bit = 1,
404 .limit_in_pages = 1,
405 .useable = 1,
406 };
407 struct n_desc_struct *desc = (void *)t->thread.tls_array;
408 desc += tls;
409 desc->a = LDT_entry_a(&ud);
410 desc->b = LDT_entry_b(&ud);
411}
412
413static inline u32 read_32bit_tls(struct task_struct *t, int tls)
414{
415 struct desc_struct *desc = (void *)t->thread.tls_array;
416 desc += tls;
417 return desc->base0 |
418 (((u32)desc->base1) << 16) |
419 (((u32)desc->base2) << 24);
420}
421
422/*
423 * This gets called before we allocate a new thread and copy
424 * the current task into it.
425 */
426void prepare_to_copy(struct task_struct *tsk)
427{
428 unlazy_fpu(tsk);
429}
430
431int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
432 unsigned long unused,
433 struct task_struct * p, struct pt_regs * regs)
434{
435 int err;
436 struct pt_regs * childregs;
437 struct task_struct *me = current;
438
Andi Kleena88cde12005-11-05 17:25:54 +0100439 childregs = ((struct pt_regs *)
Al Viro57eafdc2006-01-12 01:05:39 -0800440 (THREAD_SIZE + task_stack_page(p))) - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 *childregs = *regs;
442
443 childregs->rax = 0;
444 childregs->rsp = rsp;
Andi Kleena88cde12005-11-05 17:25:54 +0100445 if (rsp == ~0UL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 childregs->rsp = (unsigned long)childregs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447
448 p->thread.rsp = (unsigned long) childregs;
449 p->thread.rsp0 = (unsigned long) (childregs+1);
450 p->thread.userrsp = me->thread.userrsp;
451
Al Viroe4f17c42006-01-12 01:05:38 -0800452 set_tsk_thread_flag(p, TIF_FORK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453
454 p->thread.fs = me->thread.fs;
455 p->thread.gs = me->thread.gs;
456
H. J. Lufd51f662005-05-01 08:58:48 -0700457 asm("mov %%gs,%0" : "=m" (p->thread.gsindex));
458 asm("mov %%fs,%0" : "=m" (p->thread.fsindex));
459 asm("mov %%es,%0" : "=m" (p->thread.es));
460 asm("mov %%ds,%0" : "=m" (p->thread.ds));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461
462 if (unlikely(me->thread.io_bitmap_ptr != NULL)) {
463 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
464 if (!p->thread.io_bitmap_ptr) {
465 p->thread.io_bitmap_max = 0;
466 return -ENOMEM;
467 }
Andi Kleena88cde12005-11-05 17:25:54 +0100468 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr,
469 IO_BITMAP_BYTES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 }
471
472 /*
473 * Set a new TLS for the child thread?
474 */
475 if (clone_flags & CLONE_SETTLS) {
476#ifdef CONFIG_IA32_EMULATION
477 if (test_thread_flag(TIF_IA32))
478 err = ia32_child_tls(p, childregs);
479 else
480#endif
481 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
482 if (err)
483 goto out;
484 }
485 err = 0;
486out:
487 if (err && p->thread.io_bitmap_ptr) {
488 kfree(p->thread.io_bitmap_ptr);
489 p->thread.io_bitmap_max = 0;
490 }
491 return err;
492}
493
494/*
495 * This special macro can be used to load a debugging register
496 */
Jan Beulich2b514e72006-03-25 16:29:22 +0100497#define loaddebug(thread,r) set_debugreg(thread->debugreg ## r, r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498
499/*
500 * switch_to(x,y) should switch tasks from x to y.
501 *
502 * This could still be optimized:
503 * - fold all the options into a flag word and test it with a single test.
504 * - could test fs/gs bitsliced
Andi Kleen099f3182006-02-03 21:51:38 +0100505 *
506 * Kprobes not supported here. Set the probe on schedule instead.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 */
Andi Kleen099f3182006-02-03 21:51:38 +0100508__kprobes struct task_struct *
Andi Kleena88cde12005-11-05 17:25:54 +0100509__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510{
511 struct thread_struct *prev = &prev_p->thread,
512 *next = &next_p->thread;
513 int cpu = smp_processor_id();
514 struct tss_struct *tss = &per_cpu(init_tss, cpu);
515
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 /*
517 * Reload esp0, LDT and the page table pointer:
518 */
519 tss->rsp0 = next->rsp0;
520
521 /*
522 * Switch DS and ES.
523 * This won't pick up thread selector changes, but I guess that is ok.
524 */
H. J. Lufd51f662005-05-01 08:58:48 -0700525 asm volatile("mov %%es,%0" : "=m" (prev->es));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 if (unlikely(next->es | prev->es))
527 loadsegment(es, next->es);
528
H. J. Lufd51f662005-05-01 08:58:48 -0700529 asm volatile ("mov %%ds,%0" : "=m" (prev->ds));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 if (unlikely(next->ds | prev->ds))
531 loadsegment(ds, next->ds);
532
533 load_TLS(next, cpu);
534
535 /*
536 * Switch FS and GS.
537 */
538 {
539 unsigned fsindex;
540 asm volatile("movl %%fs,%0" : "=r" (fsindex));
541 /* segment register != 0 always requires a reload.
542 also reload when it has changed.
543 when prev process used 64bit base always reload
544 to avoid an information leak. */
545 if (unlikely(fsindex | next->fsindex | prev->fs)) {
546 loadsegment(fs, next->fsindex);
547 /* check if the user used a selector != 0
548 * if yes clear 64bit base, since overloaded base
549 * is always mapped to the Null selector
550 */
551 if (fsindex)
552 prev->fs = 0;
553 }
554 /* when next process has a 64bit base use it */
555 if (next->fs)
556 wrmsrl(MSR_FS_BASE, next->fs);
557 prev->fsindex = fsindex;
558 }
559 {
560 unsigned gsindex;
561 asm volatile("movl %%gs,%0" : "=r" (gsindex));
562 if (unlikely(gsindex | next->gsindex | prev->gs)) {
563 load_gs_index(next->gsindex);
564 if (gsindex)
565 prev->gs = 0;
566 }
567 if (next->gs)
568 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
569 prev->gsindex = gsindex;
570 }
571
572 /*
Jan Beulich45948d72006-03-25 16:29:25 +0100573 * Switch the PDA and FPU contexts.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 */
575 prev->userrsp = read_pda(oldrsp);
576 write_pda(oldrsp, next->userrsp);
577 write_pda(pcurrent, next_p);
Jan Beulich45948d72006-03-25 16:29:25 +0100578 /* This must be here to ensure both math_state_restore() and
579 kernel_fpu_begin() work consistently. */
580 unlazy_fpu(prev_p);
Andi Kleena88cde12005-11-05 17:25:54 +0100581 write_pda(kernelstack,
Al Viro57eafdc2006-01-12 01:05:39 -0800582 task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583
584 /*
585 * Now maybe reload the debug registers
586 */
587 if (unlikely(next->debugreg7)) {
588 loaddebug(next, 0);
589 loaddebug(next, 1);
590 loaddebug(next, 2);
591 loaddebug(next, 3);
592 /* no 4 and 5 */
593 loaddebug(next, 6);
594 loaddebug(next, 7);
595 }
596
597
598 /*
599 * Handle the IO bitmap
600 */
601 if (unlikely(prev->io_bitmap_ptr || next->io_bitmap_ptr)) {
602 if (next->io_bitmap_ptr)
603 /*
604 * Copy the relevant range of the IO bitmap.
605 * Normally this is 128 bytes or less:
606 */
607 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
608 max(prev->io_bitmap_max, next->io_bitmap_max));
609 else {
610 /*
611 * Clear any possible leftover bits:
612 */
613 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
614 }
615 }
616
617 return prev_p;
618}
619
620/*
621 * sys_execve() executes a new program.
622 */
623asmlinkage
624long sys_execve(char __user *name, char __user * __user *argv,
625 char __user * __user *envp, struct pt_regs regs)
626{
627 long error;
628 char * filename;
629
630 filename = getname(name);
631 error = PTR_ERR(filename);
632 if (IS_ERR(filename))
633 return error;
634 error = do_execve(filename, argv, envp, &regs);
635 if (error == 0) {
636 task_lock(current);
637 current->ptrace &= ~PT_DTRACE;
638 task_unlock(current);
639 }
640 putname(filename);
641 return error;
642}
643
644void set_personality_64bit(void)
645{
646 /* inherit personality from parent */
647
648 /* Make sure to be in 64bit mode */
649 clear_thread_flag(TIF_IA32);
650
651 /* TBD: overwrites user setup. Should have two bits.
652 But 64bit processes have always behaved this way,
653 so it's not too bad. The main problem is just that
654 32bit childs are affected again. */
655 current->personality &= ~READ_IMPLIES_EXEC;
656}
657
658asmlinkage long sys_fork(struct pt_regs *regs)
659{
660 return do_fork(SIGCHLD, regs->rsp, regs, 0, NULL, NULL);
661}
662
Andi Kleena88cde12005-11-05 17:25:54 +0100663asmlinkage long
664sys_clone(unsigned long clone_flags, unsigned long newsp,
665 void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666{
667 if (!newsp)
668 newsp = regs->rsp;
669 return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
670}
671
672/*
673 * This is trivial, and on the face of it looks like it
674 * could equally well be done in user mode.
675 *
676 * Not so, for quite unobvious reasons - register pressure.
677 * In user mode vfork() cannot have a stack frame, and if
678 * done by calling the "clone()" system call directly, you
679 * do not have enough call-clobbered registers to hold all
680 * the information you need.
681 */
682asmlinkage long sys_vfork(struct pt_regs *regs)
683{
684 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->rsp, regs, 0,
685 NULL, NULL);
686}
687
688unsigned long get_wchan(struct task_struct *p)
689{
690 unsigned long stack;
691 u64 fp,rip;
692 int count = 0;
693
694 if (!p || p == current || p->state==TASK_RUNNING)
695 return 0;
Al Viro57eafdc2006-01-12 01:05:39 -0800696 stack = (unsigned long)task_stack_page(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697 if (p->thread.rsp < stack || p->thread.rsp > stack+THREAD_SIZE)
698 return 0;
699 fp = *(u64 *)(p->thread.rsp);
700 do {
Andi Kleena88cde12005-11-05 17:25:54 +0100701 if (fp < (unsigned long)stack ||
702 fp > (unsigned long)stack+THREAD_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 return 0;
704 rip = *(u64 *)(fp+8);
705 if (!in_sched_functions(rip))
706 return rip;
707 fp = *(u64 *)fp;
708 } while (count++ < 16);
709 return 0;
710}
711
712long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
713{
714 int ret = 0;
715 int doit = task == current;
716 int cpu;
717
718 switch (code) {
719 case ARCH_SET_GS:
Suresh Siddha84929802005-06-21 17:14:32 -0700720 if (addr >= TASK_SIZE_OF(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 return -EPERM;
722 cpu = get_cpu();
723 /* handle small bases via the GDT because that's faster to
724 switch. */
725 if (addr <= 0xffffffff) {
726 set_32bit_tls(task, GS_TLS, addr);
727 if (doit) {
728 load_TLS(&task->thread, cpu);
729 load_gs_index(GS_TLS_SEL);
730 }
731 task->thread.gsindex = GS_TLS_SEL;
732 task->thread.gs = 0;
733 } else {
734 task->thread.gsindex = 0;
735 task->thread.gs = addr;
736 if (doit) {
Andi Kleena88cde12005-11-05 17:25:54 +0100737 load_gs_index(0);
738 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 }
740 }
741 put_cpu();
742 break;
743 case ARCH_SET_FS:
744 /* Not strictly needed for fs, but do it for symmetry
745 with gs */
Suresh Siddha84929802005-06-21 17:14:32 -0700746 if (addr >= TASK_SIZE_OF(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747 return -EPERM;
748 cpu = get_cpu();
749 /* handle small bases via the GDT because that's faster to
750 switch. */
751 if (addr <= 0xffffffff) {
752 set_32bit_tls(task, FS_TLS, addr);
753 if (doit) {
754 load_TLS(&task->thread, cpu);
Andi Kleena88cde12005-11-05 17:25:54 +0100755 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 }
757 task->thread.fsindex = FS_TLS_SEL;
758 task->thread.fs = 0;
759 } else {
760 task->thread.fsindex = 0;
761 task->thread.fs = addr;
762 if (doit) {
763 /* set the selector to 0 to not confuse
764 __switch_to */
Andi Kleena88cde12005-11-05 17:25:54 +0100765 asm volatile("movl %0,%%fs" :: "r" (0));
766 ret = checking_wrmsrl(MSR_FS_BASE, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 }
768 }
769 put_cpu();
770 break;
771 case ARCH_GET_FS: {
772 unsigned long base;
773 if (task->thread.fsindex == FS_TLS_SEL)
774 base = read_32bit_tls(task, FS_TLS);
Andi Kleena88cde12005-11-05 17:25:54 +0100775 else if (doit)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 rdmsrl(MSR_FS_BASE, base);
Andi Kleena88cde12005-11-05 17:25:54 +0100777 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778 base = task->thread.fs;
779 ret = put_user(base, (unsigned long __user *)addr);
780 break;
781 }
782 case ARCH_GET_GS: {
783 unsigned long base;
784 if (task->thread.gsindex == GS_TLS_SEL)
785 base = read_32bit_tls(task, GS_TLS);
Andi Kleena88cde12005-11-05 17:25:54 +0100786 else if (doit)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787 rdmsrl(MSR_KERNEL_GS_BASE, base);
Andi Kleena88cde12005-11-05 17:25:54 +0100788 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 base = task->thread.gs;
790 ret = put_user(base, (unsigned long __user *)addr);
791 break;
792 }
793
794 default:
795 ret = -EINVAL;
796 break;
797 }
798
799 return ret;
800}
801
802long sys_arch_prctl(int code, unsigned long addr)
803{
804 return do_arch_prctl(current, code, addr);
805}
806
807/*
808 * Capture the user space registers if the task is not running (in user space)
809 */
810int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs)
811{
812 struct pt_regs *pp, ptregs;
813
Al Virobb049232006-01-12 01:05:38 -0800814 pp = task_pt_regs(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815
816 ptregs = *pp;
817 ptregs.cs &= 0xffff;
818 ptregs.ss &= 0xffff;
819
820 elf_core_copy_regs(regs, &ptregs);
821
822 return 1;
823}
824
825unsigned long arch_align_stack(unsigned long sp)
826{
827 if (randomize_va_space)
828 sp -= get_random_int() % 8192;
829 return sp & ~0xf;
830}