blob: c535f33bbb9c9ad108c394618474bbbce1f64f86 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/kernel/fork.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7/*
8 * 'fork.c' contains the help-routines for the 'fork' system call
9 * (see also entry.S and others).
10 * Fork is rather simple, once you get the hang of it, but the memory
11 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
12 */
13
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/slab.h>
15#include <linux/init.h>
16#include <linux/unistd.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/module.h>
18#include <linux/vmalloc.h>
19#include <linux/completion.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/personality.h>
21#include <linux/mempolicy.h>
22#include <linux/sem.h>
23#include <linux/file.h>
Al Viro9f3acc32008-04-24 07:44:08 -040024#include <linux/fdtable.h>
Jens Axboeda9cbc82008-06-30 20:42:08 +020025#include <linux/iocontext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/key.h>
27#include <linux/binfmts.h>
28#include <linux/mman.h>
Andrea Arcangelicddb8a52008-07-28 15:46:29 -070029#include <linux/mmu_notifier.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <linux/fs.h>
Serge E. Hallynab516012006-10-02 02:18:06 -070031#include <linux/nsproxy.h>
Randy.Dunlapc59ede72006-01-11 12:17:46 -080032#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#include <linux/cpu.h>
Paul Menageb4f48b62007-10-18 23:39:33 -070034#include <linux/cgroup.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <linux/security.h>
Mel Gormana1e78772008-07-23 21:27:23 -070036#include <linux/hugetlb.h>
Will Drewrye2cfabdf2012-04-12 16:47:57 -050037#include <linux/seccomp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038#include <linux/swap.h>
39#include <linux/syscalls.h>
40#include <linux/jiffies.h>
41#include <linux/futex.h>
Linus Torvalds8141c7f2008-11-15 10:20:36 -080042#include <linux/compat.h>
Eric Dumazet207205a2011-03-22 16:30:44 -070043#include <linux/kthread.h>
Andrew Morton7c3ab732006-12-10 02:19:19 -080044#include <linux/task_io_accounting_ops.h>
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070045#include <linux/rcupdate.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070046#include <linux/ptrace.h>
47#include <linux/mount.h>
48#include <linux/audit.h>
Pavel Emelianov78fb7462008-02-07 00:13:51 -080049#include <linux/memcontrol.h>
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +010050#include <linux/ftrace.h>
Mike Galbraith5e2bf012012-05-10 13:01:45 -070051#include <linux/proc_fs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070052#include <linux/profile.h>
53#include <linux/rmap.h>
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070054#include <linux/ksm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <linux/acct.h>
Jay Lan8f0ab512006-09-30 23:28:59 -070056#include <linux/tsacct_kern.h>
Matt Helsley9f460802005-11-07 00:59:16 -080057#include <linux/cn_proc.h>
Rafael J. Wysockiba96a0c2007-05-23 13:57:25 -070058#include <linux/freezer.h>
Shailabh Nagarca74e922006-07-14 00:24:36 -070059#include <linux/delayacct.h>
Shailabh Nagarad4ecbc2006-07-14 00:24:44 -070060#include <linux/taskstats_kern.h>
Arjan van de Ven0a425402006-09-26 10:52:38 +020061#include <linux/random.h>
Miloslav Trmac522ed772007-07-15 23:40:56 -070062#include <linux/tty.h>
Jens Axboefd0928d2008-01-24 08:52:45 +010063#include <linux/blkdev.h>
Al Viro5ad4e532009-03-29 19:50:06 -040064#include <linux/fs_struct.h>
Eric Sandeen7c9f8862008-04-22 16:38:23 -050065#include <linux/magic.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020066#include <linux/perf_event.h>
Stanislaw Gruszka42c4ab42009-07-29 12:15:26 +020067#include <linux/posix-timers.h>
Avi Kivity8e7cac72009-11-29 16:34:48 +020068#include <linux/user-return-notifier.h>
Ying Han3d5992d2010-10-26 14:21:23 -070069#include <linux/oom.h>
Andrea Arcangeliba761492011-01-13 15:46:58 -080070#include <linux/khugepaged.h>
Oleg Nesterovd80e7312012-02-24 20:07:11 +010071#include <linux/signalfd.h>
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +053072#include <linux/uprobes.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
74#include <asm/pgtable.h>
75#include <asm/pgalloc.h>
76#include <asm/uaccess.h>
77#include <asm/mmu_context.h>
78#include <asm/cacheflush.h>
79#include <asm/tlbflush.h>
80
Steven Rostedtad8d75f2009-04-14 19:39:12 -040081#include <trace/events/sched.h>
82
KAMEZAWA Hiroyuki43d2b112012-01-10 15:08:09 -080083#define CREATE_TRACE_POINTS
84#include <trace/events/task.h>
85
Linus Torvalds1da177e2005-04-16 15:20:36 -070086/*
87 * Protected counters by write_lock_irq(&tasklist_lock)
88 */
89unsigned long total_forks; /* Handle normal Linux uptimes. */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -070090int nr_threads; /* The idle threads do not count.. */
Linus Torvalds1da177e2005-04-16 15:20:36 -070091
92int max_threads; /* tunable limit on nr_threads */
93
94DEFINE_PER_CPU(unsigned long, process_counts) = 0;
95
Christoph Hellwigc59923a2006-07-10 04:45:40 -070096__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
Paul E. McKenneydb1466b2010-03-03 07:46:56 -080097
98#ifdef CONFIG_PROVE_RCU
99int lockdep_tasklist_lock_is_held(void)
100{
101 return lockdep_is_held(&tasklist_lock);
102}
103EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
104#endif /* #ifdef CONFIG_PROVE_RCU */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105
106int nr_processes(void)
107{
108 int cpu;
109 int total = 0;
110
Ian Campbell1d510752009-11-03 10:11:14 +0000111 for_each_possible_cpu(cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 total += per_cpu(process_counts, cpu);
113
114 return total;
115}
116
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700117void __weak arch_release_task_struct(struct task_struct *tsk)
118{
119}
120
Thomas Gleixnerf5e10282012-05-05 15:05:48 +0000121#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
Christoph Lametere18b8902006-12-06 20:33:20 -0800122static struct kmem_cache *task_struct_cachep;
Thomas Gleixner41101802012-05-05 15:05:41 +0000123
124static inline struct task_struct *alloc_task_struct_node(int node)
125{
126 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
127}
128
Thomas Gleixner41101802012-05-05 15:05:41 +0000129static inline void free_task_struct(struct task_struct *tsk)
130{
Thomas Gleixner41101802012-05-05 15:05:41 +0000131 kmem_cache_free(task_struct_cachep, tsk);
132}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133#endif
134
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700135void __weak arch_release_thread_info(struct thread_info *ti)
136{
137}
138
Thomas Gleixnerf5e10282012-05-05 15:05:48 +0000139#ifndef CONFIG_ARCH_THREAD_INFO_ALLOCATOR
Thomas Gleixner41101802012-05-05 15:05:41 +0000140
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000141/*
142 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
143 * kmemcache based allocator.
144 */
145# if THREAD_SIZE >= PAGE_SIZE
Eric Dumazetb6a84012011-03-22 16:30:42 -0700146static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
147 int node)
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700148{
Glauber Costa2ad306b2012-12-18 14:22:18 -0800149 struct page *page = alloc_pages_node(node, THREADINFO_GFP_ACCOUNTED,
Thomas Gleixner2889f602012-05-05 15:05:41 +0000150 THREAD_SIZE_ORDER);
Eric Dumazetb6a84012011-03-22 16:30:42 -0700151
152 return page ? page_address(page) : NULL;
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700153}
154
155static inline void free_thread_info(struct thread_info *ti)
156{
Glauber Costa2ad306b2012-12-18 14:22:18 -0800157 free_memcg_kmem_pages((unsigned long)ti, THREAD_SIZE_ORDER);
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700158}
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000159# else
160static struct kmem_cache *thread_info_cache;
161
162static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
163 int node)
164{
165 return kmem_cache_alloc_node(thread_info_cache, THREADINFO_GFP, node);
166}
167
168static void free_thread_info(struct thread_info *ti)
169{
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000170 kmem_cache_free(thread_info_cache, ti);
171}
172
173void thread_info_cache_init(void)
174{
175 thread_info_cache = kmem_cache_create("thread_info", THREAD_SIZE,
176 THREAD_SIZE, 0, NULL);
177 BUG_ON(thread_info_cache == NULL);
178}
179# endif
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700180#endif
181
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182/* SLAB cache for signal_struct structures (tsk->signal) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800183static struct kmem_cache *signal_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184
185/* SLAB cache for sighand_struct structures (tsk->sighand) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800186struct kmem_cache *sighand_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187
188/* SLAB cache for files_struct structures (tsk->files) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800189struct kmem_cache *files_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190
191/* SLAB cache for fs_struct structures (tsk->fs) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800192struct kmem_cache *fs_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
194/* SLAB cache for vm_area_struct structures */
Christoph Lametere18b8902006-12-06 20:33:20 -0800195struct kmem_cache *vm_area_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196
197/* SLAB cache for mm_struct structures (tsk->mm) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800198static struct kmem_cache *mm_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -0700200static void account_kernel_stack(struct thread_info *ti, int account)
201{
202 struct zone *zone = page_zone(virt_to_page(ti));
203
204 mod_zone_page_state(zone, NR_KERNEL_STACK, account);
205}
206
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207void free_task(struct task_struct *tsk)
208{
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -0700209 account_kernel_stack(tsk->stack, -1);
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700210 arch_release_thread_info(tsk->stack);
Roman Zippelf7e42172007-05-09 02:35:17 -0700211 free_thread_info(tsk->stack);
Ingo Molnar23f78d4a2006-06-27 02:54:53 -0700212 rt_mutex_debug_task_free(tsk);
Frederic Weisbeckerfb526072008-11-25 21:07:04 +0100213 ftrace_graph_exit_task(tsk);
Will Drewrye2cfabdf2012-04-12 16:47:57 -0500214 put_seccomp_filter(tsk);
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700215 arch_release_task_struct(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 free_task_struct(tsk);
217}
218EXPORT_SYMBOL(free_task);
219
Oleg Nesterovea6d2902010-05-26 14:43:16 -0700220static inline void free_signal_struct(struct signal_struct *sig)
221{
Oleg Nesterov97101eb2010-05-26 14:43:20 -0700222 taskstats_tgid_free(sig);
Mike Galbraith1c5354d2011-01-05 11:16:04 +0100223 sched_autogroup_exit(sig);
Oleg Nesterovea6d2902010-05-26 14:43:16 -0700224 kmem_cache_free(signal_cachep, sig);
225}
226
227static inline void put_signal_struct(struct signal_struct *sig)
228{
Mike Galbraith1c5354d2011-01-05 11:16:04 +0100229 if (atomic_dec_and_test(&sig->sigcnt))
Oleg Nesterovea6d2902010-05-26 14:43:16 -0700230 free_signal_struct(sig);
231}
232
Andrew Morton158d9eb2006-03-31 02:31:34 -0800233void __put_task_struct(struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234{
Eugene Teo270f7222007-10-18 23:40:38 -0700235 WARN_ON(!tsk->exit_state);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 WARN_ON(atomic_read(&tsk->usage));
237 WARN_ON(tsk == current);
238
Kees Cook1a2a4d02011-12-21 12:17:03 -0800239 security_task_free(tsk);
David Howellse0e81732009-09-02 09:13:40 +0100240 exit_creds(tsk);
Shailabh Nagar35df17c2006-08-31 21:27:38 -0700241 delayacct_tsk_free(tsk);
Oleg Nesterovea6d2902010-05-26 14:43:16 -0700242 put_signal_struct(tsk->signal);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243
244 if (!profile_handoff_task(tsk))
245 free_task(tsk);
246}
Rik van Riel77c100c2011-02-01 09:51:46 -0500247EXPORT_SYMBOL_GPL(__put_task_struct);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248
Thomas Gleixner6c0a9fa2012-05-05 15:05:40 +0000249void __init __weak arch_task_cache_init(void) { }
Suresh Siddha61c46282008-03-10 15:28:04 -0700250
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251void __init fork_init(unsigned long mempages)
252{
Thomas Gleixnerf5e10282012-05-05 15:05:48 +0000253#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254#ifndef ARCH_MIN_TASKALIGN
255#define ARCH_MIN_TASKALIGN L1_CACHE_BYTES
256#endif
257 /* create a slab on which task_structs can be allocated */
258 task_struct_cachep =
259 kmem_cache_create("task_struct", sizeof(struct task_struct),
Vegard Nossum2dff4402008-05-31 15:56:17 +0200260 ARCH_MIN_TASKALIGN, SLAB_PANIC | SLAB_NOTRACK, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261#endif
262
Suresh Siddha61c46282008-03-10 15:28:04 -0700263 /* do the arch specific task caches init */
264 arch_task_cache_init();
265
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 /*
267 * The default maximum number of threads is set to a safe
268 * value: the thread structures can take up at most half
269 * of memory.
270 */
271 max_threads = mempages / (8 * THREAD_SIZE / PAGE_SIZE);
272
273 /*
274 * we need to allow at least 20 threads to boot a system
275 */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700276 if (max_threads < 20)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 max_threads = 20;
278
279 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
280 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
281 init_task.signal->rlim[RLIMIT_SIGPENDING] =
282 init_task.signal->rlim[RLIMIT_NPROC];
283}
284
Suresh Siddha61c46282008-03-10 15:28:04 -0700285int __attribute__((weak)) arch_dup_task_struct(struct task_struct *dst,
286 struct task_struct *src)
287{
288 *dst = *src;
289 return 0;
290}
291
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292static struct task_struct *dup_task_struct(struct task_struct *orig)
293{
294 struct task_struct *tsk;
295 struct thread_info *ti;
Eric Sandeen7c9f8862008-04-22 16:38:23 -0500296 unsigned long *stackend;
Eric Dumazet207205a2011-03-22 16:30:44 -0700297 int node = tsk_fork_get_node(orig);
Peter Zijlstra3e26c142007-10-16 23:25:50 -0700298 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299
Eric Dumazet504f52b2011-03-22 16:30:41 -0700300 tsk = alloc_task_struct_node(node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 if (!tsk)
302 return NULL;
303
Eric Dumazetb6a84012011-03-22 16:30:42 -0700304 ti = alloc_thread_info_node(tsk, node);
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700305 if (!ti)
306 goto free_tsk;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700308 err = arch_dup_task_struct(tsk, orig);
Suresh Siddha61c46282008-03-10 15:28:04 -0700309 if (err)
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700310 goto free_ti;
Suresh Siddha61c46282008-03-10 15:28:04 -0700311
Andrew Morton87bec582012-07-30 14:42:31 -0700312 tsk->stack = ti;
313
314 setup_thread_stack(tsk, orig);
Avi Kivity8e7cac72009-11-29 16:34:48 +0200315 clear_user_return_notifier(tsk);
Mike Galbraithf26f9af2010-12-08 11:05:42 +0100316 clear_tsk_need_resched(tsk);
Eric Sandeen7c9f8862008-04-22 16:38:23 -0500317 stackend = end_of_stack(tsk);
318 *stackend = STACK_END_MAGIC; /* for overflow detection */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319
Arjan van de Ven0a425402006-09-26 10:52:38 +0200320#ifdef CONFIG_CC_STACKPROTECTOR
321 tsk->stack_canary = get_random_int();
322#endif
323
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700324 /*
325 * One for us, one for whoever does the "release_task()" (usually
326 * parent)
327 */
328 atomic_set(&tsk->usage, 2);
Alexey Dobriyan6c5c9342006-09-29 01:59:40 -0700329#ifdef CONFIG_BLK_DEV_IO_TRACE
Jens Axboe2056a782006-03-23 20:00:26 +0100330 tsk->btrace_seq = 0;
Alexey Dobriyan6c5c9342006-09-29 01:59:40 -0700331#endif
Jens Axboea0aa7f62006-04-20 13:05:33 +0200332 tsk->splice_pipe = NULL;
Eric Dumazet5640f762012-09-23 23:04:42 +0000333 tsk->task_frag.page = NULL;
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -0700334
335 account_kernel_stack(ti, 1);
336
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337 return tsk;
Suresh Siddha61c46282008-03-10 15:28:04 -0700338
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700339free_ti:
Suresh Siddha61c46282008-03-10 15:28:04 -0700340 free_thread_info(ti);
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700341free_tsk:
Suresh Siddha61c46282008-03-10 15:28:04 -0700342 free_task_struct(tsk);
343 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344}
345
346#ifdef CONFIG_MMU
Alexey Dobriyana39bc512007-10-18 23:41:10 -0700347static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348{
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700349 struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 struct rb_node **rb_link, *rb_parent;
351 int retval;
352 unsigned long charge;
353 struct mempolicy *pol;
354
Oleg Nesterov32cdba12012-11-14 19:03:42 +0100355 uprobe_start_dup_mmap();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356 down_write(&oldmm->mmap_sem);
Ralf Baechleec8c0442006-12-12 17:14:57 +0000357 flush_cache_dup_mm(oldmm);
Oleg Nesterovf8ac4ec2012-08-08 17:11:42 +0200358 uprobe_dup_mmap(oldmm, mm);
Ingo Molnarad339452006-07-03 00:25:15 -0700359 /*
360 * Not linked in yet - no deadlock potential:
361 */
362 down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
Hugh Dickins7ee78232005-10-29 18:16:08 -0700363
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 mm->locked_vm = 0;
365 mm->mmap = NULL;
366 mm->mmap_cache = NULL;
367 mm->free_area_cache = oldmm->mmap_base;
Wolfgang Wander1363c3c2005-06-21 17:14:49 -0700368 mm->cached_hole_size = ~0UL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369 mm->map_count = 0;
Rusty Russell94894242009-03-30 22:05:12 -0600370 cpumask_clear(mm_cpumask(mm));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 mm->mm_rb = RB_ROOT;
372 rb_link = &mm->mm_rb.rb_node;
373 rb_parent = NULL;
374 pprev = &mm->mmap;
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700375 retval = ksm_fork(mm, oldmm);
376 if (retval)
377 goto out;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800378 retval = khugepaged_fork(mm, oldmm);
379 if (retval)
380 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700382 prev = NULL;
Hugh Dickinsfd3e42f2005-10-29 18:16:06 -0700383 for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 struct file *file;
385
386 if (mpnt->vm_flags & VM_DONTCOPY) {
Hugh Dickinsab50b8e2005-10-29 18:15:56 -0700387 vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
Huang Shijie44de9d02012-07-31 16:41:49 -0700388 -vma_pages(mpnt));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389 continue;
390 }
391 charge = 0;
392 if (mpnt->vm_flags & VM_ACCOUNT) {
Huang Shijieb2412b72012-07-30 14:42:30 -0700393 unsigned long len = vma_pages(mpnt);
394
Al Viro191c5422012-02-13 03:58:52 +0000395 if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 goto fail_nomem;
397 charge = len;
398 }
Christoph Lametere94b1762006-12-06 20:33:17 -0800399 tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400 if (!tmp)
401 goto fail_nomem;
402 *tmp = *mpnt;
Rik van Riel5beb4932010-03-05 13:42:07 -0800403 INIT_LIST_HEAD(&tmp->anon_vma_chain);
Lee Schermerhorn846a16b2008-04-28 02:13:09 -0700404 pol = mpol_dup(vma_policy(mpnt));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405 retval = PTR_ERR(pol);
406 if (IS_ERR(pol))
407 goto fail_nomem_policy;
408 vma_set_policy(tmp, pol);
Andrea Arcangelia247c3a2010-09-22 13:05:12 -0700409 tmp->vm_mm = mm;
Rik van Riel5beb4932010-03-05 13:42:07 -0800410 if (anon_vma_fork(tmp, mpnt))
411 goto fail_nomem_anon_vma_fork;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412 tmp->vm_flags &= ~VM_LOCKED;
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700413 tmp->vm_next = tmp->vm_prev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 file = tmp->vm_file;
415 if (file) {
Josef "Jeff" Sipekf3a43f32006-12-08 02:36:43 -0800416 struct inode *inode = file->f_path.dentry->d_inode;
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000417 struct address_space *mapping = file->f_mapping;
418
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 get_file(file);
420 if (tmp->vm_flags & VM_DENYWRITE)
421 atomic_dec(&inode->i_writecount);
Peter Zijlstra3d48ae42011-05-24 17:12:06 -0700422 mutex_lock(&mapping->i_mmap_mutex);
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000423 if (tmp->vm_flags & VM_SHARED)
424 mapping->i_mmap_writable++;
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000425 flush_dcache_mmap_lock(mapping);
426 /* insert tmp into the share list, just after mpnt */
Michel Lespinasse9826a512012-10-08 16:31:35 -0700427 if (unlikely(tmp->vm_flags & VM_NONLINEAR))
428 vma_nonlinear_insert(tmp,
429 &mapping->i_mmap_nonlinear);
430 else
431 vma_interval_tree_insert_after(tmp, mpnt,
432 &mapping->i_mmap);
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000433 flush_dcache_mmap_unlock(mapping);
Peter Zijlstra3d48ae42011-05-24 17:12:06 -0700434 mutex_unlock(&mapping->i_mmap_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 }
436
437 /*
Mel Gormana1e78772008-07-23 21:27:23 -0700438 * Clear hugetlb-related page reserves for children. This only
439 * affects MAP_PRIVATE mappings. Faults generated by the child
440 * are not guaranteed to succeed, even if read-only
441 */
442 if (is_vm_hugetlb_page(tmp))
443 reset_vma_resv_huge_pages(tmp);
444
445 /*
Hugh Dickins7ee78232005-10-29 18:16:08 -0700446 * Link in the new vma and copy the page table entries.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 *pprev = tmp;
449 pprev = &tmp->vm_next;
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700450 tmp->vm_prev = prev;
451 prev = tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452
453 __vma_link_rb(mm, tmp, rb_link, rb_parent);
454 rb_link = &tmp->vm_rb.rb_right;
455 rb_parent = &tmp->vm_rb;
456
457 mm->map_count++;
Hugh Dickins0b0db142005-11-21 21:32:20 -0800458 retval = copy_page_range(mm, oldmm, mpnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459
460 if (tmp->vm_ops && tmp->vm_ops->open)
461 tmp->vm_ops->open(tmp);
462
463 if (retval)
464 goto out;
465 }
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200466 /* a new mm has just been created */
467 arch_dup_mmap(oldmm, mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468 retval = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469out:
Hugh Dickins7ee78232005-10-29 18:16:08 -0700470 up_write(&mm->mmap_sem);
Hugh Dickinsfd3e42f2005-10-29 18:16:06 -0700471 flush_tlb_mm(oldmm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 up_write(&oldmm->mmap_sem);
Oleg Nesterov32cdba12012-11-14 19:03:42 +0100473 uprobe_end_dup_mmap();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 return retval;
Rik van Riel5beb4932010-03-05 13:42:07 -0800475fail_nomem_anon_vma_fork:
476 mpol_put(pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477fail_nomem_policy:
478 kmem_cache_free(vm_area_cachep, tmp);
479fail_nomem:
480 retval = -ENOMEM;
481 vm_unacct_memory(charge);
482 goto out;
483}
484
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700485static inline int mm_alloc_pgd(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486{
487 mm->pgd = pgd_alloc(mm);
488 if (unlikely(!mm->pgd))
489 return -ENOMEM;
490 return 0;
491}
492
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700493static inline void mm_free_pgd(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494{
Benjamin Herrenschmidt5e541972008-02-04 22:29:14 -0800495 pgd_free(mm, mm->pgd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496}
497#else
498#define dup_mmap(mm, oldmm) (0)
499#define mm_alloc_pgd(mm) (0)
500#define mm_free_pgd(mm)
501#endif /* CONFIG_MMU */
502
Daniel Walker23ff4442007-10-18 03:06:07 -0700503__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504
Christoph Lametere94b1762006-12-06 20:33:17 -0800505#define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506#define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))
507
Hidehiro Kawai4cb0e112009-01-06 14:42:47 -0800508static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
509
510static int __init coredump_filter_setup(char *s)
511{
512 default_dump_filter =
513 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
514 MMF_DUMP_FILTER_MASK;
515 return 1;
516}
517
518__setup("coredump_filter=", coredump_filter_setup);
519
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520#include <linux/init_task.h>
521
Alexey Dobriyan858f0992009-09-23 15:57:32 -0700522static void mm_init_aio(struct mm_struct *mm)
523{
524#ifdef CONFIG_AIO
525 spin_lock_init(&mm->ioctx_lock);
526 INIT_HLIST_HEAD(&mm->ioctx_list);
527#endif
528}
529
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700530static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531{
532 atomic_set(&mm->mm_users, 1);
533 atomic_set(&mm->mm_count, 1);
534 init_rwsem(&mm->mmap_sem);
535 INIT_LIST_HEAD(&mm->mmlist);
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700536 mm->flags = (current->mm) ?
537 (current->mm->flags & MMF_INIT_MASK) : default_dump_filter;
Oleg Nesterov999d9fc2008-07-25 01:47:41 -0700538 mm->core_state = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539 mm->nr_ptes = 0;
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -0800540 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541 spin_lock_init(&mm->page_table_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 mm->free_area_cache = TASK_UNMAPPED_BASE;
Wolfgang Wander1363c3c2005-06-21 17:14:49 -0700543 mm->cached_hole_size = ~0UL;
Alexey Dobriyan858f0992009-09-23 15:57:32 -0700544 mm_init_aio(mm);
Balbir Singhcf475ad2008-04-29 01:00:16 -0700545 mm_init_owner(mm, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546
547 if (likely(!mm_alloc_pgd(mm))) {
548 mm->def_flags = 0;
Andrea Arcangelicddb8a52008-07-28 15:46:29 -0700549 mmu_notifier_mm_init(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 return mm;
551 }
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800552
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 free_mm(mm);
554 return NULL;
555}
556
Konstantin Khlebnikovc3f03272012-03-21 16:33:48 -0700557static void check_mm(struct mm_struct *mm)
558{
559 int i;
560
561 for (i = 0; i < NR_MM_COUNTERS; i++) {
562 long x = atomic_long_read(&mm->rss_stat.count[i]);
563
564 if (unlikely(x))
565 printk(KERN_ALERT "BUG: Bad rss-counter state "
566 "mm:%p idx:%d val:%ld\n", mm, i, x);
567 }
568
569#ifdef CONFIG_TRANSPARENT_HUGEPAGE
570 VM_BUG_ON(mm->pmd_huge_pte);
571#endif
572}
573
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574/*
575 * Allocate and initialize an mm_struct.
576 */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700577struct mm_struct *mm_alloc(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578{
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700579 struct mm_struct *mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580
581 mm = allocate_mm();
KOSAKI Motohirode03c722011-05-24 17:12:15 -0700582 if (!mm)
583 return NULL;
584
585 memset(mm, 0, sizeof(*mm));
Linus Torvalds6345d242011-05-29 11:32:28 -0700586 mm_init_cpumask(mm);
587 return mm_init(mm, current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588}
589
590/*
591 * Called when the last reference to the mm
592 * is dropped: either by a lazy thread or by
593 * mmput. Free the page directory and the mm.
594 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800595void __mmdrop(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596{
597 BUG_ON(mm == &init_mm);
598 mm_free_pgd(mm);
599 destroy_context(mm);
Andrea Arcangelicddb8a52008-07-28 15:46:29 -0700600 mmu_notifier_mm_destroy(mm);
Konstantin Khlebnikovc3f03272012-03-21 16:33:48 -0700601 check_mm(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602 free_mm(mm);
603}
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200604EXPORT_SYMBOL_GPL(__mmdrop);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
606/*
607 * Decrement the use count and release all resources for an mm.
608 */
609void mmput(struct mm_struct *mm)
610{
Andrew Morton0ae26f12006-06-23 02:05:15 -0700611 might_sleep();
612
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 if (atomic_dec_and_test(&mm->mm_users)) {
Srikar Dronamrajud4b3b632012-03-30 23:56:31 +0530614 uprobe_clear_state(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 exit_aio(mm);
Andrea Arcangeli1c2fb7a2009-09-21 17:02:22 -0700616 ksm_exit(mm);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800617 khugepaged_exit(mm); /* must run before exit_mmap */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618 exit_mmap(mm);
Matt Helsley925d1c42008-04-29 01:01:36 -0700619 set_mm_exe_file(mm, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 if (!list_empty(&mm->mmlist)) {
621 spin_lock(&mmlist_lock);
622 list_del(&mm->mmlist);
623 spin_unlock(&mmlist_lock);
624 }
Hiroshi Shimamoto801460d2009-09-23 15:57:41 -0700625 if (mm->binfmt)
626 module_put(mm->binfmt->module);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 mmdrop(mm);
628 }
629}
630EXPORT_SYMBOL_GPL(mmput);
631
Jiri Slaby38646012011-05-26 16:25:46 -0700632void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
633{
634 if (new_exe_file)
635 get_file(new_exe_file);
636 if (mm->exe_file)
637 fput(mm->exe_file);
638 mm->exe_file = new_exe_file;
Jiri Slaby38646012011-05-26 16:25:46 -0700639}
640
641struct file *get_mm_exe_file(struct mm_struct *mm)
642{
643 struct file *exe_file;
644
Konstantin Khlebnikov2dd8ad82012-10-08 16:28:51 -0700645 /* We need mmap_sem to protect against races with removal of exe_file */
Jiri Slaby38646012011-05-26 16:25:46 -0700646 down_read(&mm->mmap_sem);
647 exe_file = mm->exe_file;
648 if (exe_file)
649 get_file(exe_file);
650 up_read(&mm->mmap_sem);
651 return exe_file;
652}
653
654static void dup_mm_exe_file(struct mm_struct *oldmm, struct mm_struct *newmm)
655{
656 /* It's safe to write the exe_file pointer without exe_file_lock because
657 * this is called during fork when the task is not yet in /proc */
658 newmm->exe_file = get_mm_exe_file(oldmm);
659}
660
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661/**
662 * get_task_mm - acquire a reference to the task's mm
663 *
Oleg Nesterov246bb0b2008-07-25 01:47:38 -0700664 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 * this kernel workthread has transiently adopted a user mm with use_mm,
666 * to do its AIO) is not set and if so returns a reference to it, after
667 * bumping up the use count. User must release the mm via mmput()
668 * after use. Typically used by /proc and ptrace.
669 */
670struct mm_struct *get_task_mm(struct task_struct *task)
671{
672 struct mm_struct *mm;
673
674 task_lock(task);
675 mm = task->mm;
676 if (mm) {
Oleg Nesterov246bb0b2008-07-25 01:47:38 -0700677 if (task->flags & PF_KTHREAD)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678 mm = NULL;
679 else
680 atomic_inc(&mm->mm_users);
681 }
682 task_unlock(task);
683 return mm;
684}
685EXPORT_SYMBOL_GPL(get_task_mm);
686
Christopher Yeoh8cdb8782012-02-02 11:34:09 +1030687struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
688{
689 struct mm_struct *mm;
690 int err;
691
692 err = mutex_lock_killable(&task->signal->cred_guard_mutex);
693 if (err)
694 return ERR_PTR(err);
695
696 mm = get_task_mm(task);
697 if (mm && mm != current->mm &&
698 !ptrace_may_access(task, mode)) {
699 mmput(mm);
700 mm = ERR_PTR(-EACCES);
701 }
702 mutex_unlock(&task->signal->cred_guard_mutex);
703
704 return mm;
705}
706
Oleg Nesterov57b59c42012-03-05 14:59:13 -0800707static void complete_vfork_done(struct task_struct *tsk)
Oleg Nesterovc415c3b2012-03-05 14:59:13 -0800708{
Oleg Nesterovd68b46f2012-03-05 14:59:13 -0800709 struct completion *vfork;
Oleg Nesterovc415c3b2012-03-05 14:59:13 -0800710
Oleg Nesterovd68b46f2012-03-05 14:59:13 -0800711 task_lock(tsk);
712 vfork = tsk->vfork_done;
713 if (likely(vfork)) {
714 tsk->vfork_done = NULL;
715 complete(vfork);
716 }
717 task_unlock(tsk);
718}
719
720static int wait_for_vfork_done(struct task_struct *child,
721 struct completion *vfork)
722{
723 int killed;
724
725 freezer_do_not_count();
726 killed = wait_for_completion_killable(vfork);
727 freezer_count();
728
729 if (killed) {
730 task_lock(child);
731 child->vfork_done = NULL;
732 task_unlock(child);
733 }
734
735 put_task_struct(child);
736 return killed;
Oleg Nesterovc415c3b2012-03-05 14:59:13 -0800737}
738
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739/* Please note the differences between mmput and mm_release.
740 * mmput is called whenever we stop holding onto a mm_struct,
741 * error success whatever.
742 *
743 * mm_release is called after a mm_struct has been removed
744 * from the current process.
745 *
746 * This difference is important for error handling, when we
747 * only half set up a mm_struct for a new process and need to restore
748 * the old one. Because we mmput the new mm_struct before
749 * restoring the old one. . .
750 * Eric Biederman 10 January 1998
751 */
752void mm_release(struct task_struct *tsk, struct mm_struct *mm)
753{
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800754 /* Get rid of any futexes when releasing the mm */
755#ifdef CONFIG_FUTEX
Peter Zijlstrafc6b1772009-10-05 18:17:32 +0200756 if (unlikely(tsk->robust_list)) {
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800757 exit_robust_list(tsk);
Peter Zijlstrafc6b1772009-10-05 18:17:32 +0200758 tsk->robust_list = NULL;
759 }
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800760#ifdef CONFIG_COMPAT
Peter Zijlstrafc6b1772009-10-05 18:17:32 +0200761 if (unlikely(tsk->compat_robust_list)) {
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800762 compat_exit_robust_list(tsk);
Peter Zijlstrafc6b1772009-10-05 18:17:32 +0200763 tsk->compat_robust_list = NULL;
764 }
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800765#endif
Thomas Gleixner322a2c12009-10-05 18:18:03 +0200766 if (unlikely(!list_empty(&tsk->pi_state_list)))
767 exit_pi_state_list(tsk);
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800768#endif
769
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +0530770 uprobe_free_utask(tsk);
771
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772 /* Get rid of any cached register state */
773 deactivate_mm(tsk, mm);
774
Roland McGrathfec1d012006-12-06 20:36:34 -0800775 /*
776 * If we're exiting normally, clear a user-space tid field if
777 * requested. We leave this alone when dying by signal, to leave
778 * the value intact in a core dump, and to save the unnecessary
Oleg Nesterovd68b46f2012-03-05 14:59:13 -0800779 * trouble, say, a killed vfork parent shouldn't touch this mm.
780 * Userland only wants this done for a sys_exit.
Roland McGrathfec1d012006-12-06 20:36:34 -0800781 */
Eric Dumazet9c8a8222009-08-06 15:09:28 -0700782 if (tsk->clear_child_tid) {
783 if (!(tsk->flags & PF_SIGNALED) &&
784 atomic_read(&mm->mm_users) > 1) {
785 /*
786 * We don't check the error code - if userspace has
787 * not set up a proper pointer then tough luck.
788 */
789 put_user(0, tsk->clear_child_tid);
790 sys_futex(tsk->clear_child_tid, FUTEX_WAKE,
791 1, NULL, NULL, 0);
792 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793 tsk->clear_child_tid = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794 }
Konstantin Khlebnikovf7505d642012-05-31 16:26:21 -0700795
796 /*
797 * All done, finally we can wake up parent and return this mm to him.
798 * Also kthread_stop() uses this completion for synchronization.
799 */
800 if (tsk->vfork_done)
801 complete_vfork_done(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802}
803
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800804/*
805 * Allocate a new mm structure and copy contents from the
806 * mm structure of the passed in task structure.
807 */
Carsten Otte402b0862008-03-25 18:47:10 +0100808struct mm_struct *dup_mm(struct task_struct *tsk)
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800809{
810 struct mm_struct *mm, *oldmm = current->mm;
811 int err;
812
813 if (!oldmm)
814 return NULL;
815
816 mm = allocate_mm();
817 if (!mm)
818 goto fail_nomem;
819
820 memcpy(mm, oldmm, sizeof(*mm));
Linus Torvalds6345d242011-05-29 11:32:28 -0700821 mm_init_cpumask(mm);
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800822
Andrea Arcangelie7a00c42011-01-13 15:46:45 -0800823#ifdef CONFIG_TRANSPARENT_HUGEPAGE
824 mm->pmd_huge_pte = NULL;
825#endif
Mel Gorman5bca2302012-11-22 14:40:03 +0000826#ifdef CONFIG_NUMA_BALANCING
827 mm->first_nid = NUMA_PTE_SCAN_INIT;
828#endif
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800829 if (!mm_init(mm, tsk))
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800830 goto fail_nomem;
831
832 if (init_new_context(tsk, mm))
833 goto fail_nocontext;
834
Matt Helsley925d1c42008-04-29 01:01:36 -0700835 dup_mm_exe_file(oldmm, mm);
836
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800837 err = dup_mmap(mm, oldmm);
838 if (err)
839 goto free_pt;
840
841 mm->hiwater_rss = get_mm_rss(mm);
842 mm->hiwater_vm = mm->total_vm;
843
Hiroshi Shimamoto801460d2009-09-23 15:57:41 -0700844 if (mm->binfmt && !try_module_get(mm->binfmt->module))
845 goto free_pt;
846
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800847 return mm;
848
849free_pt:
Hiroshi Shimamoto801460d2009-09-23 15:57:41 -0700850 /* don't put binfmt in mmput, we haven't got module yet */
851 mm->binfmt = NULL;
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800852 mmput(mm);
853
854fail_nomem:
855 return NULL;
856
857fail_nocontext:
858 /*
859 * If init_new_context() failed, we cannot use mmput() to free the mm
860 * because it calls destroy_context()
861 */
862 mm_free_pgd(mm);
863 free_mm(mm);
864 return NULL;
865}
866
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700867static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868{
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700869 struct mm_struct *mm, *oldmm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 int retval;
871
872 tsk->min_flt = tsk->maj_flt = 0;
873 tsk->nvcsw = tsk->nivcsw = 0;
Mandeep Singh Baines17406b82009-02-06 15:37:47 -0800874#ifdef CONFIG_DETECT_HUNG_TASK
875 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
876#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877
878 tsk->mm = NULL;
879 tsk->active_mm = NULL;
880
881 /*
882 * Are we cloning a kernel thread?
883 *
884 * We need to steal a active VM for that..
885 */
886 oldmm = current->mm;
887 if (!oldmm)
888 return 0;
889
890 if (clone_flags & CLONE_VM) {
891 atomic_inc(&oldmm->mm_users);
892 mm = oldmm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893 goto good_mm;
894 }
895
896 retval = -ENOMEM;
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800897 mm = dup_mm(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898 if (!mm)
899 goto fail_nomem;
900
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901good_mm:
902 tsk->mm = mm;
903 tsk->active_mm = mm;
904 return 0;
905
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906fail_nomem:
907 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908}
909
Alexey Dobriyana39bc512007-10-18 23:41:10 -0700910static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911{
Al Viro498052b2009-03-30 07:20:30 -0400912 struct fs_struct *fs = current->fs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 if (clone_flags & CLONE_FS) {
Al Viro498052b2009-03-30 07:20:30 -0400914 /* tsk->fs is already what we want */
Nick Piggin2a4419b2010-08-18 04:37:33 +1000915 spin_lock(&fs->lock);
Al Viro498052b2009-03-30 07:20:30 -0400916 if (fs->in_exec) {
Nick Piggin2a4419b2010-08-18 04:37:33 +1000917 spin_unlock(&fs->lock);
Al Viro498052b2009-03-30 07:20:30 -0400918 return -EAGAIN;
919 }
920 fs->users++;
Nick Piggin2a4419b2010-08-18 04:37:33 +1000921 spin_unlock(&fs->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922 return 0;
923 }
Al Viro498052b2009-03-30 07:20:30 -0400924 tsk->fs = copy_fs_struct(fs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925 if (!tsk->fs)
926 return -ENOMEM;
927 return 0;
928}
929
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700930static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
JANAK DESAIa016f332006-02-07 12:59:02 -0800931{
932 struct files_struct *oldf, *newf;
933 int error = 0;
934
935 /*
936 * A background process may not have any files ...
937 */
938 oldf = current->files;
939 if (!oldf)
940 goto out;
941
942 if (clone_flags & CLONE_FILES) {
943 atomic_inc(&oldf->count);
944 goto out;
945 }
946
JANAK DESAIa016f332006-02-07 12:59:02 -0800947 newf = dup_fd(oldf, &error);
948 if (!newf)
949 goto out;
950
951 tsk->files = newf;
952 error = 0;
953out:
954 return error;
955}
956
Jens Axboefadad8782008-01-24 08:54:47 +0100957static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
Jens Axboefd0928d2008-01-24 08:52:45 +0100958{
959#ifdef CONFIG_BLOCK
960 struct io_context *ioc = current->io_context;
Tejun Heo6e736be2011-12-14 00:33:38 +0100961 struct io_context *new_ioc;
Jens Axboefd0928d2008-01-24 08:52:45 +0100962
963 if (!ioc)
964 return 0;
Jens Axboefadad8782008-01-24 08:54:47 +0100965 /*
966 * Share io context with parent, if CLONE_IO is set
967 */
968 if (clone_flags & CLONE_IO) {
Tejun Heo3d487492012-03-05 13:15:25 -0800969 ioc_task_link(ioc);
970 tsk->io_context = ioc;
Jens Axboefadad8782008-01-24 08:54:47 +0100971 } else if (ioprio_valid(ioc->ioprio)) {
Tejun Heo6e736be2011-12-14 00:33:38 +0100972 new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
973 if (unlikely(!new_ioc))
Jens Axboefd0928d2008-01-24 08:52:45 +0100974 return -ENOMEM;
975
Tejun Heo6e736be2011-12-14 00:33:38 +0100976 new_ioc->ioprio = ioc->ioprio;
Tejun Heo11a31222012-02-07 07:51:30 +0100977 put_io_context(new_ioc);
Jens Axboefd0928d2008-01-24 08:52:45 +0100978 }
979#endif
980 return 0;
981}
982
Alexey Dobriyana39bc512007-10-18 23:41:10 -0700983static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984{
985 struct sighand_struct *sig;
986
Zhaolei60348802009-01-06 14:40:46 -0800987 if (clone_flags & CLONE_SIGHAND) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 atomic_inc(&current->sighand->count);
989 return 0;
990 }
991 sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
Ingo Molnare56d0902006-01-08 01:01:37 -0800992 rcu_assign_pointer(tsk->sighand, sig);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 if (!sig)
994 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995 atomic_set(&sig->count, 1);
996 memcpy(sig->action, current->sighand->action, sizeof(sig->action));
997 return 0;
998}
999
Oleg Nesterova7e53282006-03-28 16:11:27 -08001000void __cleanup_sighand(struct sighand_struct *sighand)
Oleg Nesterovc81addc2006-03-28 16:11:17 -08001001{
Oleg Nesterovd80e7312012-02-24 20:07:11 +01001002 if (atomic_dec_and_test(&sighand->count)) {
1003 signalfd_cleanup(sighand);
Oleg Nesterovc81addc2006-03-28 16:11:17 -08001004 kmem_cache_free(sighand_cachep, sighand);
Oleg Nesterovd80e7312012-02-24 20:07:11 +01001005 }
Oleg Nesterovc81addc2006-03-28 16:11:17 -08001006}
1007
Frank Mayharf06febc2008-09-12 09:54:39 -07001008
1009/*
1010 * Initialize POSIX timer handling for a thread group.
1011 */
1012static void posix_cpu_timers_init_group(struct signal_struct *sig)
1013{
Jiri Slaby78d7d402010-03-05 13:42:54 -08001014 unsigned long cpu_limit;
1015
Frank Mayharf06febc2008-09-12 09:54:39 -07001016 /* Thread group counters. */
1017 thread_group_cputime_init(sig);
1018
Jiri Slaby78d7d402010-03-05 13:42:54 -08001019 cpu_limit = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
1020 if (cpu_limit != RLIM_INFINITY) {
1021 sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit);
Oleg Nesterov6279a7512009-03-27 01:06:07 +01001022 sig->cputimer.running = 1;
1023 }
1024
Frank Mayharf06febc2008-09-12 09:54:39 -07001025 /* The timer lists. */
1026 INIT_LIST_HEAD(&sig->cpu_timers[0]);
1027 INIT_LIST_HEAD(&sig->cpu_timers[1]);
1028 INIT_LIST_HEAD(&sig->cpu_timers[2]);
1029}
1030
Alexey Dobriyana39bc512007-10-18 23:41:10 -07001031static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032{
1033 struct signal_struct *sig;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034
Oleg Nesterov4ab6c082009-08-26 14:29:24 -07001035 if (clone_flags & CLONE_THREAD)
Peter Zijlstra490dea42008-11-24 17:06:57 +01001036 return 0;
Oleg Nesterov6279a7512009-03-27 01:06:07 +01001037
Veaceslav Falicoa56704e2010-03-10 15:23:01 -08001038 sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 tsk->signal = sig;
1040 if (!sig)
1041 return -ENOMEM;
1042
Oleg Nesterovb3ac0222010-05-26 14:43:24 -07001043 sig->nr_threads = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044 atomic_set(&sig->live, 1);
Oleg Nesterovb3ac0222010-05-26 14:43:24 -07001045 atomic_set(&sig->sigcnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046 init_waitqueue_head(&sig->wait_chldexit);
Oleg Nesterovdb51aec2008-04-30 00:52:52 -07001047 sig->curr_target = tsk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048 init_sigpending(&sig->shared_pending);
1049 INIT_LIST_HEAD(&sig->posix_timers);
1050
Thomas Gleixnerc9cb2e32007-02-16 01:27:49 -08001051 hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 sig->real_timer.function = it_real_fn;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001053
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054 task_lock(current->group_leader);
1055 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
1056 task_unlock(current->group_leader);
1057
Oleg Nesterov6279a7512009-03-27 01:06:07 +01001058 posix_cpu_timers_init_group(sig);
1059
Miloslav Trmac522ed772007-07-15 23:40:56 -07001060 tty_audit_fork(sig);
Mike Galbraith5091faa2010-11-30 14:18:03 +01001061 sched_autogroup_fork(sig);
Miloslav Trmac522ed772007-07-15 23:40:56 -07001062
Ben Blum4714d1d2011-05-26 16:25:18 -07001063#ifdef CONFIG_CGROUPS
Tejun Heo257058a2011-12-12 18:12:21 -08001064 init_rwsem(&sig->group_rwsem);
Ben Blum4714d1d2011-05-26 16:25:18 -07001065#endif
1066
David Rientjesa63d83f2010-08-09 17:19:46 -07001067 sig->oom_score_adj = current->signal->oom_score_adj;
Mandeep Singh Bainesdabb16f2011-01-13 15:46:05 -08001068 sig->oom_score_adj_min = current->signal->oom_score_adj_min;
KOSAKI Motohiro28b83c52009-09-21 17:03:13 -07001069
Lennart Poetteringebec18a2012-03-23 15:01:54 -07001070 sig->has_child_subreaper = current->signal->has_child_subreaper ||
1071 current->signal->is_child_subreaper;
1072
KOSAKI Motohiro9b1bf122010-10-27 15:34:08 -07001073 mutex_init(&sig->cred_guard_mutex);
1074
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075 return 0;
1076}
1077
Alexey Dobriyana39bc512007-10-18 23:41:10 -07001078static void copy_flags(unsigned long clone_flags, struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079{
1080 unsigned long new_flags = p->flags;
1081
Tejun Heo21aa9af2010-06-08 21:40:37 +02001082 new_flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 new_flags |= PF_FORKNOEXEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001084 p->flags = new_flags;
1085}
1086
Heiko Carstens17da2bd2009-01-14 14:14:10 +01001087SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088{
1089 current->clear_child_tid = tidptr;
1090
Pavel Emelyanovb4888932007-10-18 23:40:14 -07001091 return task_pid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092}
1093
Alexey Dobriyana39bc512007-10-18 23:41:10 -07001094static void rt_mutex_init_task(struct task_struct *p)
Ingo Molnar23f78d4a2006-06-27 02:54:53 -07001095{
Thomas Gleixner1d615482009-11-17 14:54:03 +01001096 raw_spin_lock_init(&p->pi_lock);
Zilvinas Valinskase29e1752007-03-16 13:38:34 -08001097#ifdef CONFIG_RT_MUTEXES
Dima Zavin732375c2011-07-07 17:27:59 -07001098 plist_head_init(&p->pi_waiters);
Ingo Molnar23f78d4a2006-06-27 02:54:53 -07001099 p->pi_blocked_on = NULL;
Ingo Molnar23f78d4a2006-06-27 02:54:53 -07001100#endif
1101}
1102
Balbir Singhcf475ad2008-04-29 01:00:16 -07001103#ifdef CONFIG_MM_OWNER
1104void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
1105{
1106 mm->owner = p;
1107}
1108#endif /* CONFIG_MM_OWNER */
1109
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110/*
Frank Mayharf06febc2008-09-12 09:54:39 -07001111 * Initialize POSIX timer handling for a single task.
1112 */
1113static void posix_cpu_timers_init(struct task_struct *tsk)
1114{
Martin Schwidefsky64861632011-12-15 14:56:09 +01001115 tsk->cputime_expires.prof_exp = 0;
1116 tsk->cputime_expires.virt_exp = 0;
Frank Mayharf06febc2008-09-12 09:54:39 -07001117 tsk->cputime_expires.sched_exp = 0;
1118 INIT_LIST_HEAD(&tsk->cpu_timers[0]);
1119 INIT_LIST_HEAD(&tsk->cpu_timers[1]);
1120 INIT_LIST_HEAD(&tsk->cpu_timers[2]);
1121}
1122
1123/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124 * This creates a new process as a copy of the old one,
1125 * but does not actually start it yet.
1126 *
1127 * It copies the registers, and all the appropriate
1128 * parts of the process environment (as per the clone
1129 * flags). The actual kick-off is left to the caller.
1130 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07001131static struct task_struct *copy_process(unsigned long clone_flags,
1132 unsigned long stack_start,
Ingo Molnar36c8b582006-07-03 00:25:41 -07001133 unsigned long stack_size,
Ingo Molnar36c8b582006-07-03 00:25:41 -07001134 int __user *child_tidptr,
Roland McGrath09a05392008-07-25 19:45:47 -07001135 struct pid *pid,
1136 int trace)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137{
1138 int retval;
Mariusz Kozlowskia24efe62007-10-18 23:41:09 -07001139 struct task_struct *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140
1141 if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
1142 return ERR_PTR(-EINVAL);
1143
1144 /*
1145 * Thread groups must share signals as well, and detached threads
1146 * can only be started up within the thread group.
1147 */
1148 if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
1149 return ERR_PTR(-EINVAL);
1150
1151 /*
1152 * Shared signal handlers imply shared VM. By way of the above,
1153 * thread groups also imply shared VM. Blocking this case allows
1154 * for various simplifications in other code.
1155 */
1156 if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
1157 return ERR_PTR(-EINVAL);
1158
Sukadev Bhattiprolu123be072009-09-23 15:57:20 -07001159 /*
1160 * Siblings of global init remain as zombies on exit since they are
1161 * not reaped by their parent (swapper). To solve this and to avoid
1162 * multi-rooted process trees, prevent global and container-inits
1163 * from creating siblings.
1164 */
1165 if ((clone_flags & CLONE_PARENT) &&
1166 current->signal->flags & SIGNAL_UNKILLABLE)
1167 return ERR_PTR(-EINVAL);
1168
Eric W. Biederman8382fca2012-12-20 19:26:06 -08001169 /*
1170 * If the new process will be in a different pid namespace
1171 * don't allow the creation of threads.
1172 */
1173 if ((clone_flags & (CLONE_VM|CLONE_NEWPID)) &&
1174 (task_active_pid_ns(current) != current->nsproxy->pid_ns))
1175 return ERR_PTR(-EINVAL);
1176
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177 retval = security_task_create(clone_flags);
1178 if (retval)
1179 goto fork_out;
1180
1181 retval = -ENOMEM;
1182 p = dup_task_struct(current);
1183 if (!p)
1184 goto fork_out;
1185
Steven Rostedtf7e8b612009-06-02 16:39:48 -04001186 ftrace_graph_init_task(p);
Will Drewrye2cfabdf2012-04-12 16:47:57 -05001187 get_seccomp_filter(p);
Steven Rostedtf7e8b612009-06-02 16:39:48 -04001188
Peter Zijlstrabea493a2006-10-17 00:10:33 -07001189 rt_mutex_init_task(p);
1190
Ingo Molnard12c1a32008-07-14 12:09:28 +02001191#ifdef CONFIG_PROVE_LOCKING
Ingo Molnarde30a2b2006-07-03 00:24:42 -07001192 DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
1193 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1194#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001195 retval = -EAGAIN;
David Howells3b11a1d2008-11-14 10:39:26 +11001196 if (atomic_read(&p->real_cred->user->processes) >=
Jiri Slaby78d7d402010-03-05 13:42:54 -08001197 task_rlimit(p, RLIMIT_NPROC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198 if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) &&
Serge Hallyn18b6e042008-10-15 16:38:45 -05001199 p->real_cred->user != INIT_USER)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200 goto bad_fork_free;
1201 }
Vasiliy Kulikov72fa5992011-08-08 19:02:04 +04001202 current->flags &= ~PF_NPROC_EXCEEDED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001203
David Howellsf1752ee2008-11-14 10:39:17 +11001204 retval = copy_creds(p, clone_flags);
1205 if (retval < 0)
1206 goto bad_fork_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207
1208 /*
1209 * If multiple threads are within copy_process(), then this check
1210 * triggers too late. This doesn't hurt, the check is only there
1211 * to stop root fork bombs.
1212 */
Li Zefan04ec93f2009-02-06 08:17:19 +00001213 retval = -EAGAIN;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214 if (nr_threads >= max_threads)
1215 goto bad_fork_cleanup_count;
1216
Al Viroa1261f52005-11-13 16:06:55 -08001217 if (!try_module_get(task_thread_info(p)->exec_domain->module))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218 goto bad_fork_cleanup_count;
1219
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 p->did_exec = 0;
Shailabh Nagarca74e922006-07-14 00:24:36 -07001221 delayacct_tsk_init(p); /* Must remain after dup_task_struct() */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222 copy_flags(clone_flags, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223 INIT_LIST_HEAD(&p->children);
1224 INIT_LIST_HEAD(&p->sibling);
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07001225 rcu_copy_process(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226 p->vfork_done = NULL;
1227 spin_lock_init(&p->alloc_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228
Linus Torvalds1da177e2005-04-16 15:20:36 -07001229 init_sigpending(&p->pending);
1230
Martin Schwidefsky64861632011-12-15 14:56:09 +01001231 p->utime = p->stime = p->gtime = 0;
1232 p->utimescaled = p->stimescaled = 0;
Hidetoshi Setod99ca3b2009-12-02 17:26:47 +09001233#ifndef CONFIG_VIRT_CPU_ACCOUNTING
Frederic Weisbeckerd37f761d2012-11-22 00:58:35 +01001234 p->prev_cputime.utime = p->prev_cputime.stime = 0;
Hidetoshi Setod99ca3b2009-12-02 17:26:47 +09001235#endif
KAMEZAWA Hiroyukia3a2e762010-04-06 14:34:42 -07001236#if defined(SPLIT_RSS_COUNTING)
1237 memset(&p->rss_stat, 0, sizeof(p->rss_stat));
1238#endif
Balbir Singh172ba842007-07-09 18:52:00 +02001239
Arjan van de Ven69766752008-09-01 15:52:40 -07001240 p->default_timer_slack_ns = current->timer_slack_ns;
1241
Andrea Righi59954772008-07-27 17:29:15 +02001242 task_io_accounting_init(&p->ioac);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243 acct_clear_integrals(p);
1244
Frank Mayharf06febc2008-09-12 09:54:39 -07001245 posix_cpu_timers_init(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001246
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247 do_posix_clock_monotonic_gettime(&p->start_time);
Tomas Janousek924b42d2007-07-15 23:39:42 -07001248 p->real_start_time = p->start_time;
1249 monotonic_to_bootbased(&p->real_start_time);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250 p->io_context = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251 p->audit_context = NULL;
Ben Blum4714d1d2011-05-26 16:25:18 -07001252 if (clone_flags & CLONE_THREAD)
Tejun Heo257058a2011-12-12 18:12:21 -08001253 threadgroup_change_begin(current);
Paul Menageb4f48b62007-10-18 23:39:33 -07001254 cgroup_fork(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255#ifdef CONFIG_NUMA
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001256 p->mempolicy = mpol_dup(p->mempolicy);
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001257 if (IS_ERR(p->mempolicy)) {
1258 retval = PTR_ERR(p->mempolicy);
1259 p->mempolicy = NULL;
1260 goto bad_fork_cleanup_cgroup;
1261 }
Paul Jacksonc61afb12006-03-24 03:16:08 -08001262 mpol_fix_fork_child_flag(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263#endif
Michal Hocko778d3b02011-07-26 16:08:30 -07001264#ifdef CONFIG_CPUSETS
1265 p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
1266 p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
Mel Gormancc9a6c82012-03-21 16:34:11 -07001267 seqcount_init(&p->mems_allowed_seq);
Michal Hocko778d3b02011-07-26 16:08:30 -07001268#endif
Ingo Molnarde30a2b2006-07-03 00:24:42 -07001269#ifdef CONFIG_TRACE_IRQFLAGS
1270 p->irq_events = 0;
1271 p->hardirqs_enabled = 0;
1272 p->hardirq_enable_ip = 0;
1273 p->hardirq_enable_event = 0;
1274 p->hardirq_disable_ip = _THIS_IP_;
1275 p->hardirq_disable_event = 0;
1276 p->softirqs_enabled = 1;
1277 p->softirq_enable_ip = _THIS_IP_;
1278 p->softirq_enable_event = 0;
1279 p->softirq_disable_ip = 0;
1280 p->softirq_disable_event = 0;
1281 p->hardirq_context = 0;
1282 p->softirq_context = 0;
1283#endif
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07001284#ifdef CONFIG_LOCKDEP
1285 p->lockdep_depth = 0; /* no locks held yet */
1286 p->curr_chain_key = 0;
1287 p->lockdep_recursion = 0;
1288#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001289
Ingo Molnar408894e2006-01-09 15:59:20 -08001290#ifdef CONFIG_DEBUG_MUTEXES
1291 p->blocked_on = NULL; /* not blocked yet */
1292#endif
Andrew Mortonc255a452012-07-31 16:43:02 -07001293#ifdef CONFIG_MEMCG
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08001294 p->memcg_batch.do_batch = 0;
1295 p->memcg_batch.memcg = NULL;
1296#endif
Markus Metzger0f481402009-04-03 16:43:48 +02001297
Srivatsa Vaddagiri3c90e6e2007-11-09 22:39:39 +01001298 /* Perform scheduler related setup. Assign this task to a CPU. */
Samir Bellabes3e51e3e2011-05-11 18:18:05 +02001299 sched_fork(p);
Peter Zijlstra6ab423e2009-05-25 14:45:27 +02001300
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001301 retval = perf_event_init_task(p);
Peter Zijlstra6ab423e2009-05-25 14:45:27 +02001302 if (retval)
1303 goto bad_fork_cleanup_policy;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001304 retval = audit_alloc(p);
1305 if (retval)
David Howellsf1752ee2008-11-14 10:39:17 +11001306 goto bad_fork_cleanup_policy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307 /* copy all the process information */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001308 retval = copy_semundo(clone_flags, p);
1309 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310 goto bad_fork_cleanup_audit;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001311 retval = copy_files(clone_flags, p);
1312 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313 goto bad_fork_cleanup_semundo;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001314 retval = copy_fs(clone_flags, p);
1315 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316 goto bad_fork_cleanup_files;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001317 retval = copy_sighand(clone_flags, p);
1318 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319 goto bad_fork_cleanup_fs;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001320 retval = copy_signal(clone_flags, p);
1321 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322 goto bad_fork_cleanup_sighand;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001323 retval = copy_mm(clone_flags, p);
1324 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325 goto bad_fork_cleanup_signal;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001326 retval = copy_namespaces(clone_flags, p);
1327 if (retval)
David Howellsd84f4f92008-11-14 10:39:23 +11001328 goto bad_fork_cleanup_mm;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001329 retval = copy_io(clone_flags, p);
1330 if (retval)
Jens Axboefd0928d2008-01-24 08:52:45 +01001331 goto bad_fork_cleanup_namespaces;
Al Viroafa86fc2012-10-22 22:51:14 -04001332 retval = copy_thread(clone_flags, stack_start, stack_size, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333 if (retval)
Jens Axboefd0928d2008-01-24 08:52:45 +01001334 goto bad_fork_cleanup_io;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07001336 if (pid != &init_struct_pid) {
1337 retval = -ENOMEM;
Eric W. Biederman61bce0f2009-01-07 18:08:49 -08001338 pid = alloc_pid(p->nsproxy->pid_ns);
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07001339 if (!pid)
Jens Axboefd0928d2008-01-24 08:52:45 +01001340 goto bad_fork_cleanup_io;
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07001341 }
1342
1343 p->pid = pid_nr(pid);
1344 p->tgid = p->pid;
1345 if (clone_flags & CLONE_THREAD)
1346 p->tgid = current->tgid;
1347
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
1349 /*
1350 * Clear TID on mm_release()?
1351 */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001352 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL;
Jens Axboe73c10102011-03-08 13:19:51 +01001353#ifdef CONFIG_BLOCK
1354 p->plug = NULL;
1355#endif
Alexey Dobriyan42b2dd02007-10-16 23:27:30 -07001356#ifdef CONFIG_FUTEX
Ingo Molnar8f17d3a2006-03-27 01:16:27 -08001357 p->robust_list = NULL;
1358#ifdef CONFIG_COMPAT
1359 p->compat_robust_list = NULL;
1360#endif
Ingo Molnarc87e2832006-06-27 02:54:58 -07001361 INIT_LIST_HEAD(&p->pi_state_list);
1362 p->pi_state_cache = NULL;
Alexey Dobriyan42b2dd02007-10-16 23:27:30 -07001363#endif
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +05301364 uprobe_copy_process(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365 /*
GOTO Masanorif9a38792006-03-13 21:20:44 -08001366 * sigaltstack should be cleared when sharing the same VM
1367 */
1368 if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
1369 p->sas_ss_sp = p->sas_ss_size = 0;
1370
1371 /*
Oleg Nesterov65808072009-12-15 16:47:16 -08001372 * Syscall tracing and stepping should be turned off in the
1373 * child regardless of CLONE_PTRACE.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374 */
Oleg Nesterov65808072009-12-15 16:47:16 -08001375 user_disable_single_step(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376 clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
Laurent Viviered75e8d2005-09-03 15:57:18 -07001377#ifdef TIF_SYSCALL_EMU
1378 clear_tsk_thread_flag(p, TIF_SYSCALL_EMU);
1379#endif
Arjan van de Ven97455122008-01-25 21:08:34 +01001380 clear_all_latency_tracing(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382 /* ok, now we should be set up.. */
Oleg Nesterov5f8aadd2012-03-14 19:55:38 +01001383 if (clone_flags & CLONE_THREAD)
1384 p->exit_signal = -1;
1385 else if (clone_flags & CLONE_PARENT)
1386 p->exit_signal = current->group_leader->exit_signal;
1387 else
1388 p->exit_signal = (clone_flags & CSIGNAL);
1389
Linus Torvalds1da177e2005-04-16 15:20:36 -07001390 p->pdeath_signal = 0;
1391 p->exit_state = 0;
1392
Wu Fengguang9d823e82011-06-11 18:10:12 -06001393 p->nr_dirtied = 0;
1394 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
Wu Fengguang83712352011-06-11 19:25:42 -06001395 p->dirty_paused_when = 0;
Wu Fengguang9d823e82011-06-11 18:10:12 -06001396
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397 /*
1398 * Ok, make it visible to the rest of the system.
1399 * We dont wake it up yet.
1400 */
1401 p->group_leader = p;
Oleg Nesterov47e65322006-03-28 16:11:25 -08001402 INIT_LIST_HEAD(&p->thread_group);
Al Viro158e1642012-06-27 09:24:13 +04001403 p->task_works = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001404
1405 /* Need tasklist lock for parent etc handling! */
1406 write_lock_irq(&tasklist_lock);
1407
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408 /* CLONE_PARENT re-uses the old parent */
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01001409 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410 p->real_parent = current->real_parent;
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01001411 p->parent_exec_id = current->parent_exec_id;
1412 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413 p->real_parent = current;
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01001414 p->parent_exec_id = current->self_exec_id;
1415 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001416
Oleg Nesterov3f17da62006-02-15 22:13:24 +03001417 spin_lock(&current->sighand->siglock);
Oleg Nesterov4a2c7a72006-03-28 16:11:26 -08001418
1419 /*
1420 * Process group and session signals need to be delivered to just the
1421 * parent before the fork or both the parent and the child after the
1422 * fork. Restart if a signal comes in before we add the new process to
1423 * it's process group.
1424 * A fatal signal pending means that current will exit, so the new
1425 * thread can't slip out of an OOM kill (or normal SIGKILL).
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001426 */
Daniel Walker23ff4442007-10-18 03:06:07 -07001427 recalc_sigpending();
Oleg Nesterov4a2c7a72006-03-28 16:11:26 -08001428 if (signal_pending(current)) {
1429 spin_unlock(&current->sighand->siglock);
1430 write_unlock_irq(&tasklist_lock);
1431 retval = -ERESTARTNOINTR;
Steven Rostedtf7e8b612009-06-02 16:39:48 -04001432 goto bad_fork_free_pid;
Oleg Nesterov4a2c7a72006-03-28 16:11:26 -08001433 }
1434
Linus Torvalds1da177e2005-04-16 15:20:36 -07001435 if (clone_flags & CLONE_THREAD) {
Oleg Nesterovb3ac0222010-05-26 14:43:24 -07001436 current->signal->nr_threads++;
Oleg Nesterov4ab6c082009-08-26 14:29:24 -07001437 atomic_inc(&current->signal->live);
Oleg Nesterovb3ac0222010-05-26 14:43:24 -07001438 atomic_inc(&current->signal->sigcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001439 p->group_leader = current->group_leader;
Oleg Nesterov47e65322006-03-28 16:11:25 -08001440 list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441 }
1442
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08001443 if (likely(p->pid)) {
Tejun Heo4b9d33e2011-06-17 16:50:38 +02001444 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08001446 if (thread_group_leader(p)) {
Eric W. Biederman1c4042c2010-07-12 17:10:36 -07001447 if (is_child_reaper(pid)) {
Eric W. Biederman17cf22c2010-03-02 14:51:53 -08001448 ns_of_pid(pid)->child_reaper = p;
Eric W. Biederman1c4042c2010-07-12 17:10:36 -07001449 p->signal->flags |= SIGNAL_UNKILLABLE;
1450 }
Oleg Nesterovc97d9892006-03-28 16:11:06 -08001451
Oleg Nesterovfea9d172008-02-08 04:19:19 -08001452 p->signal->leader_pid = pid;
Alan Cox9c9f4de2008-10-13 10:37:26 +01001453 p->signal->tty = tty_kref_get(current->signal->tty);
Eric W. Biederman5cd17562007-12-04 23:45:04 -08001454 attach_pid(p, PIDTYPE_PGID, task_pgrp(current));
1455 attach_pid(p, PIDTYPE_SID, task_session(current));
Oleg Nesterov9cd80bb2009-12-17 15:27:15 -08001456 list_add_tail(&p->sibling, &p->real_parent->children);
Eric W. Biederman5e85d4a2006-04-18 22:20:16 -07001457 list_add_tail_rcu(&p->tasks, &init_task.tasks);
Christoph Lameter909ea962010-12-08 16:22:55 +01001458 __this_cpu_inc(process_counts);
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08001459 }
Sukadev Bhattiprolu85868992007-05-10 22:23:03 -07001460 attach_pid(p, PIDTYPE_PID, pid);
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08001461 nr_threads++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462 }
1463
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464 total_forks++;
Oleg Nesterov3f17da62006-02-15 22:13:24 +03001465 spin_unlock(&current->sighand->siglock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466 write_unlock_irq(&tasklist_lock);
Andrew Mortonc13cf852005-11-28 13:43:48 -08001467 proc_fork_connector(p);
Paul Menage817929e2007-10-18 23:39:36 -07001468 cgroup_post_fork(p);
Ben Blum4714d1d2011-05-26 16:25:18 -07001469 if (clone_flags & CLONE_THREAD)
Tejun Heo257058a2011-12-12 18:12:21 -08001470 threadgroup_change_end(current);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001471 perf_event_fork(p);
KAMEZAWA Hiroyuki43d2b112012-01-10 15:08:09 -08001472
1473 trace_task_newtask(p, clone_flags);
1474
Linus Torvalds1da177e2005-04-16 15:20:36 -07001475 return p;
1476
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07001477bad_fork_free_pid:
1478 if (pid != &init_struct_pid)
1479 free_pid(pid);
Jens Axboefd0928d2008-01-24 08:52:45 +01001480bad_fork_cleanup_io:
Louis Rillingb69f2292009-12-04 14:52:42 +01001481 if (p->io_context)
1482 exit_io_context(p);
Serge E. Hallynab516012006-10-02 02:18:06 -07001483bad_fork_cleanup_namespaces:
Linus Torvalds444f3782007-01-30 13:35:18 -08001484 exit_task_namespaces(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485bad_fork_cleanup_mm:
David Rientjesc9f01242011-10-31 17:07:15 -07001486 if (p->mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487 mmput(p->mm);
1488bad_fork_cleanup_signal:
Oleg Nesterov4ab6c082009-08-26 14:29:24 -07001489 if (!(clone_flags & CLONE_THREAD))
Mike Galbraith1c5354d2011-01-05 11:16:04 +01001490 free_signal_struct(p->signal);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491bad_fork_cleanup_sighand:
Oleg Nesterova7e53282006-03-28 16:11:27 -08001492 __cleanup_sighand(p->sighand);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001493bad_fork_cleanup_fs:
1494 exit_fs(p); /* blocking */
1495bad_fork_cleanup_files:
1496 exit_files(p); /* blocking */
1497bad_fork_cleanup_semundo:
1498 exit_sem(p);
1499bad_fork_cleanup_audit:
1500 audit_free(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501bad_fork_cleanup_policy:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001502 perf_event_free_task(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503#ifdef CONFIG_NUMA
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001504 mpol_put(p->mempolicy);
Paul Menageb4f48b62007-10-18 23:39:33 -07001505bad_fork_cleanup_cgroup:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506#endif
Ben Blum4714d1d2011-05-26 16:25:18 -07001507 if (clone_flags & CLONE_THREAD)
Tejun Heo257058a2011-12-12 18:12:21 -08001508 threadgroup_change_end(current);
Tejun Heo5edee612012-10-16 15:03:14 -07001509 cgroup_exit(p, 0);
Shailabh Nagar35df17c2006-08-31 21:27:38 -07001510 delayacct_tsk_free(p);
Al Viroa1261f52005-11-13 16:06:55 -08001511 module_put(task_thread_info(p)->exec_domain->module);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001512bad_fork_cleanup_count:
David Howellsd84f4f92008-11-14 10:39:23 +11001513 atomic_dec(&p->cred->user->processes);
David Howellse0e81732009-09-02 09:13:40 +01001514 exit_creds(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515bad_fork_free:
1516 free_task(p);
Oleg Nesterovfe7d37d2006-01-08 01:04:02 -08001517fork_out:
1518 return ERR_PTR(retval);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519}
1520
Oleg Nesterovf106eee2010-05-26 14:44:11 -07001521static inline void init_idle_pids(struct pid_link *links)
1522{
1523 enum pid_type type;
1524
1525 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
1526 INIT_HLIST_NODE(&links[type].node); /* not really needed */
1527 links[type].pid = &init_struct_pid;
1528 }
1529}
1530
Al Viro9abcf402007-02-01 13:52:48 +00001531struct task_struct * __cpuinit fork_idle(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532{
Ingo Molnar36c8b582006-07-03 00:25:41 -07001533 struct task_struct *task;
Al Viro62e791c2012-10-22 22:52:26 -04001534 task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0);
Oleg Nesterovf106eee2010-05-26 14:44:11 -07001535 if (!IS_ERR(task)) {
1536 init_idle_pids(task->pids);
Akinobu Mita753ca4f2006-11-25 11:09:34 -08001537 init_idle(task, cpu);
Oleg Nesterovf106eee2010-05-26 14:44:11 -07001538 }
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08001539
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540 return task;
1541}
1542
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543/*
1544 * Ok, this is the main fork-routine.
1545 *
1546 * It copies the process, and if successful kick-starts
1547 * it and waits for it to finish using the VM if required.
1548 */
1549long do_fork(unsigned long clone_flags,
1550 unsigned long stack_start,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551 unsigned long stack_size,
1552 int __user *parent_tidptr,
1553 int __user *child_tidptr)
1554{
1555 struct task_struct *p;
1556 int trace = 0;
Eric W. Biederman92476d72006-03-31 02:31:42 -08001557 long nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001558
Andrew Mortonbdff7462008-02-04 22:27:22 -08001559 /*
Serge Hallyn18b6e042008-10-15 16:38:45 -05001560 * Do some preliminary argument and permissions checking before we
1561 * actually start allocating stuff
1562 */
Eric W. Biederman50804fe2010-03-02 15:41:50 -08001563 if (clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) {
1564 if (clone_flags & (CLONE_THREAD|CLONE_PARENT))
Serge Hallyn18b6e042008-10-15 16:38:45 -05001565 return -EINVAL;
Serge Hallyn18b6e042008-10-15 16:38:45 -05001566 }
1567
1568 /*
Tejun Heo4b9d33e2011-06-17 16:50:38 +02001569 * Determine whether and which event to report to ptracer. When
1570 * called from kernel_thread or CLONE_UNTRACED is explicitly
1571 * requested, no event is reported; otherwise, report if the event
1572 * for the type of forking is enabled.
Roland McGrath09a05392008-07-25 19:45:47 -07001573 */
Al Viroe80d6662012-10-22 23:10:08 -04001574 if (!(clone_flags & CLONE_UNTRACED)) {
Tejun Heo4b9d33e2011-06-17 16:50:38 +02001575 if (clone_flags & CLONE_VFORK)
1576 trace = PTRACE_EVENT_VFORK;
1577 else if ((clone_flags & CSIGNAL) != SIGCHLD)
1578 trace = PTRACE_EVENT_CLONE;
1579 else
1580 trace = PTRACE_EVENT_FORK;
1581
1582 if (likely(!ptrace_event_enabled(current, trace)))
1583 trace = 0;
1584 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585
Al Viro62e791c2012-10-22 22:52:26 -04001586 p = copy_process(clone_flags, stack_start, stack_size,
Roland McGrath09a05392008-07-25 19:45:47 -07001587 child_tidptr, NULL, trace);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588 /*
1589 * Do this prior waking up the new thread - the thread pointer
1590 * might get invalid after that point, if the thread exits quickly.
1591 */
1592 if (!IS_ERR(p)) {
1593 struct completion vfork;
1594
Mathieu Desnoyers0a16b602008-07-18 12:16:17 -04001595 trace_sched_process_fork(current, p);
1596
Pavel Emelyanov6c5f3e72008-02-08 04:19:20 -08001597 nr = task_pid_vnr(p);
Pavel Emelyanov30e49c22007-10-18 23:40:10 -07001598
1599 if (clone_flags & CLONE_PARENT_SETTID)
1600 put_user(nr, parent_tidptr);
Sukadev Bhattiprolua6f5e062007-10-18 23:39:53 -07001601
Linus Torvalds1da177e2005-04-16 15:20:36 -07001602 if (clone_flags & CLONE_VFORK) {
1603 p->vfork_done = &vfork;
1604 init_completion(&vfork);
Oleg Nesterovd68b46f2012-03-05 14:59:13 -08001605 get_task_struct(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606 }
1607
Samir Bellabes3e51e3e2011-05-11 18:18:05 +02001608 wake_up_new_task(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001609
Tejun Heo4b9d33e2011-06-17 16:50:38 +02001610 /* forking complete and child started to run, tell ptracer */
1611 if (unlikely(trace))
1612 ptrace_event(trace, nr);
Roland McGrath09a05392008-07-25 19:45:47 -07001613
Linus Torvalds1da177e2005-04-16 15:20:36 -07001614 if (clone_flags & CLONE_VFORK) {
Oleg Nesterovd68b46f2012-03-05 14:59:13 -08001615 if (!wait_for_vfork_done(p, &vfork))
1616 ptrace_event(PTRACE_EVENT_VFORK_DONE, nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617 }
1618 } else {
Eric W. Biederman92476d72006-03-31 02:31:42 -08001619 nr = PTR_ERR(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001620 }
Eric W. Biederman92476d72006-03-31 02:31:42 -08001621 return nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622}
1623
Al Viro2aa3a7f2012-09-21 19:55:31 -04001624/*
1625 * Create a kernel thread.
1626 */
1627pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
1628{
Al Viroe80d6662012-10-22 23:10:08 -04001629 return do_fork(flags|CLONE_VM|CLONE_UNTRACED, (unsigned long)fn,
Al Viro2aa3a7f2012-09-21 19:55:31 -04001630 (unsigned long)arg, NULL, NULL);
1631}
Al Viro2aa3a7f2012-09-21 19:55:31 -04001632
Al Virod2125042012-10-23 13:17:59 -04001633#ifdef __ARCH_WANT_SYS_FORK
1634SYSCALL_DEFINE0(fork)
1635{
1636#ifdef CONFIG_MMU
Al Viroe80d6662012-10-22 23:10:08 -04001637 return do_fork(SIGCHLD, 0, 0, NULL, NULL);
Al Virod2125042012-10-23 13:17:59 -04001638#else
1639 /* can not support in nommu mode */
1640 return(-EINVAL);
1641#endif
1642}
1643#endif
1644
1645#ifdef __ARCH_WANT_SYS_VFORK
1646SYSCALL_DEFINE0(vfork)
1647{
Al Viroe80d6662012-10-22 23:10:08 -04001648 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, 0,
Al Virod2125042012-10-23 13:17:59 -04001649 0, NULL, NULL);
1650}
1651#endif
1652
1653#ifdef __ARCH_WANT_SYS_CLONE
1654#ifdef CONFIG_CLONE_BACKWARDS
1655SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
1656 int __user *, parent_tidptr,
1657 int, tls_val,
1658 int __user *, child_tidptr)
1659#elif defined(CONFIG_CLONE_BACKWARDS2)
1660SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
1661 int __user *, parent_tidptr,
1662 int __user *, child_tidptr,
1663 int, tls_val)
1664#else
1665SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
1666 int __user *, parent_tidptr,
1667 int __user *, child_tidptr,
1668 int, tls_val)
1669#endif
1670{
Al Virob1e03182013-01-19 22:13:34 -05001671 long ret = do_fork(clone_flags, newsp, 0, parent_tidptr, child_tidptr);
1672 asmlinkage_protect(5, ret, clone_flags, newsp,
1673 parent_tidptr, child_tidptr, tls_val);
1674 return ret;
Al Virod2125042012-10-23 13:17:59 -04001675}
1676#endif
1677
Ravikiran G Thirumalai5fd63b32006-01-11 22:46:15 +01001678#ifndef ARCH_MIN_MMSTRUCT_ALIGN
1679#define ARCH_MIN_MMSTRUCT_ALIGN 0
1680#endif
1681
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07001682static void sighand_ctor(void *data)
Oleg Nesterovaa1757f2006-03-28 16:11:12 -08001683{
1684 struct sighand_struct *sighand = data;
1685
Christoph Lametera35afb82007-05-16 22:10:57 -07001686 spin_lock_init(&sighand->siglock);
Davide Libenzib8fceee2007-09-20 12:40:16 -07001687 init_waitqueue_head(&sighand->signalfd_wqh);
Oleg Nesterovaa1757f2006-03-28 16:11:12 -08001688}
1689
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690void __init proc_caches_init(void)
1691{
1692 sighand_cachep = kmem_cache_create("sighand_cache",
1693 sizeof(struct sighand_struct), 0,
Vegard Nossum2dff4402008-05-31 15:56:17 +02001694 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU|
1695 SLAB_NOTRACK, sighand_ctor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001696 signal_cachep = kmem_cache_create("signal_cache",
1697 sizeof(struct signal_struct), 0,
Vegard Nossum2dff4402008-05-31 15:56:17 +02001698 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
Paul Mundt20c2df82007-07-20 10:11:58 +09001699 files_cachep = kmem_cache_create("files_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700 sizeof(struct files_struct), 0,
Vegard Nossum2dff4402008-05-31 15:56:17 +02001701 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
Paul Mundt20c2df82007-07-20 10:11:58 +09001702 fs_cachep = kmem_cache_create("fs_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001703 sizeof(struct fs_struct), 0,
Vegard Nossum2dff4402008-05-31 15:56:17 +02001704 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
Linus Torvalds6345d242011-05-29 11:32:28 -07001705 /*
1706 * FIXME! The "sizeof(struct mm_struct)" currently includes the
1707 * whole struct cpumask for the OFFSTACK case. We could change
1708 * this to *only* allocate as much of it as required by the
1709 * maximum number of CPU's we can ever have. The cpumask_allocation
1710 * is at the end of the structure, exactly for that reason.
1711 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001712 mm_cachep = kmem_cache_create("mm_struct",
Ravikiran G Thirumalai5fd63b32006-01-11 22:46:15 +01001713 sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN,
Vegard Nossum2dff4402008-05-31 15:56:17 +02001714 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
David Howells33e5d7692009-04-02 16:56:32 -07001715 vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC);
David Howells8feae132009-01-08 12:04:47 +00001716 mmap_init();
Al Viro66577192011-06-28 15:41:10 -04001717 nsproxy_cache_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718}
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001719
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001720/*
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001721 * Check constraints on flags passed to the unshare system call.
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001722 */
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001723static int check_unshare_flags(unsigned long unshare_flags)
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001724{
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001725 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
1726 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
Eric W. Biederman50804fe2010-03-02 15:41:50 -08001727 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET|
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07001728 CLONE_NEWUSER|CLONE_NEWPID))
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001729 return -EINVAL;
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001730 /*
1731 * Not implemented, but pretend it works if there is nothing to
1732 * unshare. Note that unsharing CLONE_THREAD or CLONE_SIGHAND
1733 * needs to unshare vm.
1734 */
1735 if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
1736 /* FIXME: get_task_mm() increments ->mm_users */
1737 if (atomic_read(&current->mm->mm_users) > 1)
1738 return -EINVAL;
1739 }
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001740
1741 return 0;
1742}
1743
1744/*
JANAK DESAI99d14192006-02-07 12:58:59 -08001745 * Unshare the filesystem structure if it is being shared
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001746 */
1747static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
1748{
1749 struct fs_struct *fs = current->fs;
1750
Al Viro498052b2009-03-30 07:20:30 -04001751 if (!(unshare_flags & CLONE_FS) || !fs)
1752 return 0;
1753
1754 /* don't need lock here; in the worst case we'll do useless copy */
1755 if (fs->users == 1)
1756 return 0;
1757
1758 *new_fsp = copy_fs_struct(fs);
1759 if (!*new_fsp)
1760 return -ENOMEM;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001761
1762 return 0;
1763}
1764
1765/*
JANAK DESAIa016f332006-02-07 12:59:02 -08001766 * Unshare file descriptor table if it is being shared
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001767 */
1768static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
1769{
1770 struct files_struct *fd = current->files;
JANAK DESAIa016f332006-02-07 12:59:02 -08001771 int error = 0;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001772
1773 if ((unshare_flags & CLONE_FILES) &&
JANAK DESAIa016f332006-02-07 12:59:02 -08001774 (fd && atomic_read(&fd->count) > 1)) {
1775 *new_fdp = dup_fd(fd, &error);
1776 if (!*new_fdp)
1777 return error;
1778 }
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001779
1780 return 0;
1781}
1782
1783/*
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001784 * unshare allows a process to 'unshare' part of the process
1785 * context which was originally shared using clone. copy_*
1786 * functions used by do_fork() cannot be used here directly
1787 * because they modify an inactive task_struct that is being
1788 * constructed. Here we are modifying the current, active,
1789 * task_struct.
1790 */
Heiko Carstens6559eed82009-01-14 14:14:32 +01001791SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001792{
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001793 struct fs_struct *fs, *new_fs = NULL;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001794 struct files_struct *fd, *new_fd = NULL;
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07001795 struct cred *new_cred = NULL;
Pavel Emelyanovcf7b7082007-10-18 23:39:54 -07001796 struct nsproxy *new_nsproxy = NULL;
Manfred Spraul9edff4a2008-04-29 01:00:57 -07001797 int do_sysvsem = 0;
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001798 int err;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001799
Eric W. Biederman50804fe2010-03-02 15:41:50 -08001800 /*
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07001801 * If unsharing a user namespace must also unshare the thread.
1802 */
1803 if (unshare_flags & CLONE_NEWUSER)
1804 unshare_flags |= CLONE_THREAD;
1805 /*
Eric W. Biederman50804fe2010-03-02 15:41:50 -08001806 * If unsharing a pid namespace must also unshare the thread.
1807 */
1808 if (unshare_flags & CLONE_NEWPID)
1809 unshare_flags |= CLONE_THREAD;
1810 /*
1811 * If unsharing a thread from a thread group, must also unshare vm.
1812 */
1813 if (unshare_flags & CLONE_THREAD)
1814 unshare_flags |= CLONE_VM;
1815 /*
1816 * If unsharing vm, must also unshare signal handlers.
1817 */
1818 if (unshare_flags & CLONE_VM)
1819 unshare_flags |= CLONE_SIGHAND;
Manfred Spraul6013f672008-04-29 01:00:59 -07001820 /*
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001821 * If unsharing namespace, must also unshare filesystem information.
1822 */
1823 if (unshare_flags & CLONE_NEWNS)
1824 unshare_flags |= CLONE_FS;
Eric W. Biederman50804fe2010-03-02 15:41:50 -08001825
1826 err = check_unshare_flags(unshare_flags);
1827 if (err)
1828 goto bad_unshare_out;
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001829 /*
Manfred Spraul6013f672008-04-29 01:00:59 -07001830 * CLONE_NEWIPC must also detach from the undolist: after switching
1831 * to a new ipc namespace, the semaphore arrays from the old
1832 * namespace are unreachable.
1833 */
1834 if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
Manfred Spraul9edff4a2008-04-29 01:00:57 -07001835 do_sysvsem = 1;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001836 err = unshare_fs(unshare_flags, &new_fs);
1837 if (err)
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001838 goto bad_unshare_out;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001839 err = unshare_fd(unshare_flags, &new_fd);
1840 if (err)
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001841 goto bad_unshare_cleanup_fs;
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07001842 err = unshare_userns(unshare_flags, &new_cred);
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001843 if (err)
Manfred Spraul9edff4a2008-04-29 01:00:57 -07001844 goto bad_unshare_cleanup_fd;
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07001845 err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
1846 new_cred, new_fs);
1847 if (err)
1848 goto bad_unshare_cleanup_cred;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001849
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07001850 if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
Manfred Spraul9edff4a2008-04-29 01:00:57 -07001851 if (do_sysvsem) {
1852 /*
1853 * CLONE_SYSVSEM is equivalent to sys_exit().
1854 */
1855 exit_sem(current);
1856 }
Serge E. Hallynab516012006-10-02 02:18:06 -07001857
Serge Hallync0b2fc32006-10-02 02:18:18 -07001858 if (new_nsproxy) {
Pavel Emelyanovcf7b7082007-10-18 23:39:54 -07001859 switch_task_namespaces(current, new_nsproxy);
1860 new_nsproxy = NULL;
Serge Hallync0b2fc32006-10-02 02:18:18 -07001861 }
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001862
Pavel Emelyanovcf7b7082007-10-18 23:39:54 -07001863 task_lock(current);
1864
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001865 if (new_fs) {
1866 fs = current->fs;
Nick Piggin2a4419b2010-08-18 04:37:33 +10001867 spin_lock(&fs->lock);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001868 current->fs = new_fs;
Al Viro498052b2009-03-30 07:20:30 -04001869 if (--fs->users)
1870 new_fs = NULL;
1871 else
1872 new_fs = fs;
Nick Piggin2a4419b2010-08-18 04:37:33 +10001873 spin_unlock(&fs->lock);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001874 }
1875
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001876 if (new_fd) {
1877 fd = current->files;
1878 current->files = new_fd;
1879 new_fd = fd;
1880 }
1881
1882 task_unlock(current);
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07001883
1884 if (new_cred) {
1885 /* Install the new user namespace */
1886 commit_creds(new_cred);
1887 new_cred = NULL;
1888 }
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001889 }
1890
Serge Hallync0b2fc32006-10-02 02:18:18 -07001891 if (new_nsproxy)
Linus Torvalds444f3782007-01-30 13:35:18 -08001892 put_nsproxy(new_nsproxy);
Serge Hallync0b2fc32006-10-02 02:18:18 -07001893
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07001894bad_unshare_cleanup_cred:
1895 if (new_cred)
1896 put_cred(new_cred);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001897bad_unshare_cleanup_fd:
1898 if (new_fd)
1899 put_files_struct(new_fd);
1900
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001901bad_unshare_cleanup_fs:
1902 if (new_fs)
Al Viro498052b2009-03-30 07:20:30 -04001903 free_fs_struct(new_fs);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001904
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001905bad_unshare_out:
1906 return err;
1907}
Al Viro3b125382008-04-22 05:31:30 -04001908
1909/*
1910 * Helper to unshare the files of the current task.
1911 * We don't want to expose copy_files internals to
1912 * the exec layer of the kernel.
1913 */
1914
1915int unshare_files(struct files_struct **displaced)
1916{
1917 struct task_struct *task = current;
Al Viro50704512008-04-26 05:25:00 +01001918 struct files_struct *copy = NULL;
Al Viro3b125382008-04-22 05:31:30 -04001919 int error;
1920
1921 error = unshare_fd(CLONE_FILES, &copy);
1922 if (error || !copy) {
1923 *displaced = NULL;
1924 return error;
1925 }
1926 *displaced = task->files;
1927 task_lock(task);
1928 task->files = copy;
1929 task_unlock(task);
1930 return 0;
1931}