blob: 7a5c0b229c6ae17fbf9b448952c89def3ca97e4c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/oom_kill.c
3 *
4 * Copyright (C) 1998,2000 Rik van Riel
5 * Thanks go out to Claus Fischer for some serious inspiration and
6 * for goading me into coding this file...
David Rientjesa63d83f2010-08-09 17:19:46 -07007 * Copyright (C) 2010 Google, Inc.
8 * Rewritten by David Rientjes
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 *
10 * The routines in this file are used to kill a process when
Paul Jacksona49335c2005-09-06 15:18:09 -070011 * we're seriously out of memory. This gets called from __alloc_pages()
12 * in mm/page_alloc.c when we really run out of memory.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
14 * Since we won't call these routines often (on a well-configured
15 * machine) this file will double as a 'coding guide' and a signpost
16 * for newbie kernel hackers. It features several pointers to major
17 * kernel subsystems and hints as to where to find out what things do.
18 */
19
Alexey Dobriyan8ac773b2006-10-19 23:28:32 -070020#include <linux/oom.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/mm.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040022#include <linux/err.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090023#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/sched.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010025#include <linux/sched/mm.h>
Ingo Molnarf7ccbae2017-02-08 18:51:30 +010026#include <linux/sched/coredump.h>
Ingo Molnar29930022017-02-08 18:51:36 +010027#include <linux/sched/task.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/swap.h>
29#include <linux/timex.h>
30#include <linux/jiffies.h>
Paul Jacksonef08e3b2005-09-06 15:18:13 -070031#include <linux/cpuset.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040032#include <linux/export.h>
Martin Schwidefsky8bc719d2006-09-25 23:31:20 -070033#include <linux/notifier.h>
Pavel Emelianovc7ba5c92008-02-07 00:13:58 -080034#include <linux/memcontrol.h>
David Rientjes6f48d0eb2010-08-09 17:18:52 -070035#include <linux/mempolicy.h>
David Howells5cd9c582008-08-14 11:37:28 +010036#include <linux/security.h>
David Rientjesedd45542011-03-22 16:30:12 -070037#include <linux/ptrace.h>
David Rientjesf660daa2011-10-31 17:07:07 -070038#include <linux/freezer.h>
KAMEZAWA Hiroyuki43d2b112012-01-10 15:08:09 -080039#include <linux/ftrace.h>
David Rientjesdc3f21e2012-03-21 16:33:47 -070040#include <linux/ratelimit.h>
Michal Hockoaac45362016-03-25 14:20:24 -070041#include <linux/kthread.h>
42#include <linux/init.h>
Michal Hocko4d4bbd82017-10-03 16:14:50 -070043#include <linux/mmu_notifier.h>
Michal Hockoaac45362016-03-25 14:20:24 -070044
45#include <asm/tlb.h>
46#include "internal.h"
KAMEZAWA Hiroyuki43d2b112012-01-10 15:08:09 -080047
48#define CREATE_TRACE_POINTS
49#include <trace/events/oom.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
KAMEZAWA Hiroyukifadd8fb2006-06-23 02:03:13 -070051int sysctl_panic_on_oom;
David Rientjesfe071d72007-10-16 23:25:56 -070052int sysctl_oom_kill_allocating_task;
David Rientjesad915c42010-08-09 17:18:53 -070053int sysctl_oom_dump_tasks = 1;
Johannes Weinerdc564012015-06-24 16:57:19 -070054
55DEFINE_MUTEX(oom_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
David Rientjes6f48d0eb2010-08-09 17:18:52 -070057#ifdef CONFIG_NUMA
58/**
59 * has_intersects_mems_allowed() - check task eligiblity for kill
Oleg Nesterovad962442014-01-21 15:50:00 -080060 * @start: task struct of which task to consider
David Rientjes6f48d0eb2010-08-09 17:18:52 -070061 * @mask: nodemask passed to page allocator for mempolicy ooms
62 *
63 * Task eligibility is determined by whether or not a candidate task, @tsk,
64 * shares the same mempolicy nodes as current if it is bound by such a policy
65 * and whether or not it has the same set of allowed cpuset nodes.
KOSAKI Motohiro495789a2009-09-21 17:03:14 -070066 */
Oleg Nesterovad962442014-01-21 15:50:00 -080067static bool has_intersects_mems_allowed(struct task_struct *start,
David Rientjes6f48d0eb2010-08-09 17:18:52 -070068 const nodemask_t *mask)
KOSAKI Motohiro495789a2009-09-21 17:03:14 -070069{
Oleg Nesterovad962442014-01-21 15:50:00 -080070 struct task_struct *tsk;
71 bool ret = false;
KOSAKI Motohiro495789a2009-09-21 17:03:14 -070072
Oleg Nesterovad962442014-01-21 15:50:00 -080073 rcu_read_lock();
Oleg Nesterov1da4db02014-01-21 15:49:58 -080074 for_each_thread(start, tsk) {
David Rientjes6f48d0eb2010-08-09 17:18:52 -070075 if (mask) {
76 /*
77 * If this is a mempolicy constrained oom, tsk's
78 * cpuset is irrelevant. Only return true if its
79 * mempolicy intersects current, otherwise it may be
80 * needlessly killed.
81 */
Oleg Nesterovad962442014-01-21 15:50:00 -080082 ret = mempolicy_nodemask_intersects(tsk, mask);
David Rientjes6f48d0eb2010-08-09 17:18:52 -070083 } else {
84 /*
85 * This is not a mempolicy constrained oom, so only
86 * check the mems of tsk's cpuset.
87 */
Oleg Nesterovad962442014-01-21 15:50:00 -080088 ret = cpuset_mems_allowed_intersects(current, tsk);
David Rientjes6f48d0eb2010-08-09 17:18:52 -070089 }
Oleg Nesterovad962442014-01-21 15:50:00 -080090 if (ret)
91 break;
Oleg Nesterov1da4db02014-01-21 15:49:58 -080092 }
Oleg Nesterovad962442014-01-21 15:50:00 -080093 rcu_read_unlock();
KOSAKI Motohirodf1090a2010-08-09 17:19:39 -070094
Oleg Nesterovad962442014-01-21 15:50:00 -080095 return ret;
KOSAKI Motohiro495789a2009-09-21 17:03:14 -070096}
David Rientjes6f48d0eb2010-08-09 17:18:52 -070097#else
98static bool has_intersects_mems_allowed(struct task_struct *tsk,
99 const nodemask_t *mask)
100{
101 return true;
102}
103#endif /* CONFIG_NUMA */
KOSAKI Motohiro495789a2009-09-21 17:03:14 -0700104
David Rientjes6f48d0eb2010-08-09 17:18:52 -0700105/*
106 * The process p may have detached its own ->mm while exiting or through
107 * use_mm(), but one or more of its subthreads may still have a valid
108 * pointer. Return p, or any of its subthreads with a valid ->mm, with
109 * task_lock() held.
110 */
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -0700111struct task_struct *find_lock_task_mm(struct task_struct *p)
Oleg Nesterovdd8e8f42010-08-09 17:18:45 -0700112{
Oleg Nesterov1da4db02014-01-21 15:49:58 -0800113 struct task_struct *t;
Oleg Nesterovdd8e8f42010-08-09 17:18:45 -0700114
Oleg Nesterov4d4048b2014-01-21 15:50:01 -0800115 rcu_read_lock();
116
Oleg Nesterov1da4db02014-01-21 15:49:58 -0800117 for_each_thread(p, t) {
Oleg Nesterovdd8e8f42010-08-09 17:18:45 -0700118 task_lock(t);
119 if (likely(t->mm))
Oleg Nesterov4d4048b2014-01-21 15:50:01 -0800120 goto found;
Oleg Nesterovdd8e8f42010-08-09 17:18:45 -0700121 task_unlock(t);
Oleg Nesterov1da4db02014-01-21 15:49:58 -0800122 }
Oleg Nesterov4d4048b2014-01-21 15:50:01 -0800123 t = NULL;
124found:
125 rcu_read_unlock();
Oleg Nesterovdd8e8f42010-08-09 17:18:45 -0700126
Oleg Nesterov4d4048b2014-01-21 15:50:01 -0800127 return t;
Oleg Nesterovdd8e8f42010-08-09 17:18:45 -0700128}
129
Yaowei Baidb2a0dd2015-11-06 16:28:06 -0800130/*
131 * order == -1 means the oom kill is required by sysrq, otherwise only
132 * for display purposes.
133 */
134static inline bool is_sysrq_oom(struct oom_control *oc)
135{
136 return oc->order == -1;
137}
138
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700139static inline bool is_memcg_oom(struct oom_control *oc)
140{
141 return oc->memcg != NULL;
142}
143
KOSAKI Motohiroab290ad2010-08-09 17:19:35 -0700144/* return true if the task is not adequate as candidate victim task. */
David Rientjese85bfd32010-09-22 13:05:10 -0700145static bool oom_unkillable_task(struct task_struct *p,
Johannes Weiner2314b422014-12-10 15:44:33 -0800146 struct mem_cgroup *memcg, const nodemask_t *nodemask)
KOSAKI Motohiroab290ad2010-08-09 17:19:35 -0700147{
148 if (is_global_init(p))
149 return true;
150 if (p->flags & PF_KTHREAD)
151 return true;
152
153 /* When mem_cgroup_out_of_memory() and p is not member of the group */
Johannes Weiner72835c82012-01-12 17:18:32 -0800154 if (memcg && !task_in_mem_cgroup(p, memcg))
KOSAKI Motohiroab290ad2010-08-09 17:19:35 -0700155 return true;
156
157 /* p may not have freeable memory in nodemask */
158 if (!has_intersects_mems_allowed(p, nodemask))
159 return true;
160
161 return false;
162}
163
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164/**
David Rientjesa63d83f2010-08-09 17:19:46 -0700165 * oom_badness - heuristic function to determine which candidate task to kill
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 * @p: task struct of which task we should calculate
David Rientjesa63d83f2010-08-09 17:19:46 -0700167 * @totalpages: total present RAM allowed for page allocation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 *
David Rientjesa63d83f2010-08-09 17:19:46 -0700169 * The heuristic for determining which task to kill is made to be as simple and
170 * predictable as possible. The goal is to return the highest value for the
171 * task consuming the most memory to avoid subsequent oom failures.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 */
David Rientjesa7f638f2012-05-29 15:06:47 -0700173unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *memcg,
174 const nodemask_t *nodemask, unsigned long totalpages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175{
David Rientjes1e11ad82012-06-08 13:21:26 -0700176 long points;
David Rientjes61eafb02012-06-20 12:52:58 -0700177 long adj;
KOSAKI Motohiro28b83c52009-09-21 17:03:13 -0700178
Johannes Weiner72835c82012-01-12 17:18:32 -0800179 if (oom_unkillable_task(p, memcg, nodemask))
KOSAKI Motohiro26ebc982010-08-09 17:19:37 -0700180 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
Oleg Nesterovdd8e8f42010-08-09 17:18:45 -0700182 p = find_lock_task_mm(p);
183 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 return 0;
185
Michal Hockobb8a4b72016-05-20 16:57:18 -0700186 /*
187 * Do not even consider tasks which are explicitly marked oom
Michal Hockob18dc5f2016-07-28 15:44:46 -0700188 * unkillable or have been already oom reaped or the are in
189 * the middle of vfork
Michal Hockobb8a4b72016-05-20 16:57:18 -0700190 */
David Rientjesa9c58b902012-12-11 16:02:54 -0800191 adj = (long)p->signal->oom_score_adj;
Michal Hockobb8a4b72016-05-20 16:57:18 -0700192 if (adj == OOM_SCORE_ADJ_MIN ||
Michal Hocko862e3072016-10-07 16:58:57 -0700193 test_bit(MMF_OOM_SKIP, &p->mm->flags) ||
Michal Hockob18dc5f2016-07-28 15:44:46 -0700194 in_vfork(p)) {
Michal Hocko5aecc852011-11-15 14:36:07 -0800195 task_unlock(p);
196 return 0;
197 }
198
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 /*
David Rientjesa63d83f2010-08-09 17:19:46 -0700200 * The baseline for the badness score is the proportion of RAM that each
KOSAKI Motohirof755a042011-04-27 15:26:50 -0700201 * task's rss, pagetable and swap space use.
David Rientjesa63d83f2010-08-09 17:19:46 -0700202 */
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800203 points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) +
204 atomic_long_read(&p->mm->nr_ptes) + mm_nr_pmds(p->mm);
Andrew Morton97c2c9b82006-04-18 22:20:38 -0700205 task_unlock(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206
207 /*
David Rientjesa63d83f2010-08-09 17:19:46 -0700208 * Root processes get 3% bonus, just like the __vm_enough_memory()
209 * implementation used by LSMs.
Hugh Dickins7ba34852007-01-05 16:37:03 -0800210 */
David Rientjesa63d83f2010-08-09 17:19:46 -0700211 if (has_capability_noaudit(p, CAP_SYS_ADMIN))
David Rientjes778c14a2014-01-30 15:46:11 -0800212 points -= (points * 3) / 100;
Hugh Dickins7ba34852007-01-05 16:37:03 -0800213
David Rientjes61eafb02012-06-20 12:52:58 -0700214 /* Normalize to oom_score_adj units */
215 adj *= totalpages / 1000;
216 points += adj;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
David Rientjesf19e8aa2010-09-22 13:04:52 -0700218 /*
David Rientjesa7f638f2012-05-29 15:06:47 -0700219 * Never return 0 for an eligible task regardless of the root bonus and
220 * oom_score_adj (oom_score_adj can't be OOM_SCORE_ADJ_MIN here).
David Rientjesf19e8aa2010-09-22 13:04:52 -0700221 */
David Rientjes1e11ad82012-06-08 13:21:26 -0700222 return points > 0 ? points : 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223}
224
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700225enum oom_constraint {
226 CONSTRAINT_NONE,
227 CONSTRAINT_CPUSET,
228 CONSTRAINT_MEMORY_POLICY,
229 CONSTRAINT_MEMCG,
230};
231
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232/*
Christoph Lameter9b0f8b02006-02-20 18:27:52 -0800233 * Determine the type of allocation constraint.
234 */
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700235static enum oom_constraint constrained_alloc(struct oom_control *oc)
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -0800236{
Mel Gorman54a6eb52008-04-28 02:12:16 -0700237 struct zone *zone;
Mel Gormandd1a2392008-04-28 02:12:17 -0700238 struct zoneref *z;
David Rientjes6e0fc462015-09-08 15:00:36 -0700239 enum zone_type high_zoneidx = gfp_zone(oc->gfp_mask);
David Rientjesa63d83f2010-08-09 17:19:46 -0700240 bool cpuset_limited = false;
241 int nid;
Christoph Lameter9b0f8b02006-02-20 18:27:52 -0800242
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700243 if (is_memcg_oom(oc)) {
244 oc->totalpages = mem_cgroup_get_limit(oc->memcg) ?: 1;
245 return CONSTRAINT_MEMCG;
246 }
247
David Rientjesa63d83f2010-08-09 17:19:46 -0700248 /* Default to all available memory */
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700249 oc->totalpages = totalram_pages + total_swap_pages;
250
251 if (!IS_ENABLED(CONFIG_NUMA))
252 return CONSTRAINT_NONE;
David Rientjesa63d83f2010-08-09 17:19:46 -0700253
David Rientjes6e0fc462015-09-08 15:00:36 -0700254 if (!oc->zonelist)
David Rientjesa63d83f2010-08-09 17:19:46 -0700255 return CONSTRAINT_NONE;
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -0800256 /*
257 * Reach here only when __GFP_NOFAIL is used. So, we should avoid
258 * to kill current.We have to random task kill in this case.
259 * Hopefully, CONSTRAINT_THISNODE...but no way to handle it, now.
260 */
David Rientjes6e0fc462015-09-08 15:00:36 -0700261 if (oc->gfp_mask & __GFP_THISNODE)
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -0800262 return CONSTRAINT_NONE;
Christoph Lameter9b0f8b02006-02-20 18:27:52 -0800263
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -0800264 /*
David Rientjesa63d83f2010-08-09 17:19:46 -0700265 * This is not a __GFP_THISNODE allocation, so a truncated nodemask in
266 * the page allocator means a mempolicy is in effect. Cpuset policy
267 * is enforced in get_page_from_freelist().
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -0800268 */
David Rientjes6e0fc462015-09-08 15:00:36 -0700269 if (oc->nodemask &&
270 !nodes_subset(node_states[N_MEMORY], *oc->nodemask)) {
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700271 oc->totalpages = total_swap_pages;
David Rientjes6e0fc462015-09-08 15:00:36 -0700272 for_each_node_mask(nid, *oc->nodemask)
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700273 oc->totalpages += node_spanned_pages(nid);
Christoph Lameter9b0f8b02006-02-20 18:27:52 -0800274 return CONSTRAINT_MEMORY_POLICY;
David Rientjesa63d83f2010-08-09 17:19:46 -0700275 }
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -0800276
277 /* Check this allocation failure is caused by cpuset's wall function */
David Rientjes6e0fc462015-09-08 15:00:36 -0700278 for_each_zone_zonelist_nodemask(zone, z, oc->zonelist,
279 high_zoneidx, oc->nodemask)
280 if (!cpuset_zone_allowed(zone, oc->gfp_mask))
David Rientjesa63d83f2010-08-09 17:19:46 -0700281 cpuset_limited = true;
Christoph Lameter9b0f8b02006-02-20 18:27:52 -0800282
David Rientjesa63d83f2010-08-09 17:19:46 -0700283 if (cpuset_limited) {
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700284 oc->totalpages = total_swap_pages;
David Rientjesa63d83f2010-08-09 17:19:46 -0700285 for_each_node_mask(nid, cpuset_current_mems_allowed)
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700286 oc->totalpages += node_spanned_pages(nid);
David Rientjesa63d83f2010-08-09 17:19:46 -0700287 return CONSTRAINT_CPUSET;
288 }
Christoph Lameter9b0f8b02006-02-20 18:27:52 -0800289 return CONSTRAINT_NONE;
290}
291
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700292static int oom_evaluate_task(struct task_struct *task, void *arg)
David Rientjes462607e2012-07-31 16:43:40 -0700293{
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700294 struct oom_control *oc = arg;
295 unsigned long points;
296
David Rientjes6e0fc462015-09-08 15:00:36 -0700297 if (oom_unkillable_task(task, NULL, oc->nodemask))
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700298 goto next;
David Rientjes462607e2012-07-31 16:43:40 -0700299
300 /*
301 * This task already has access to memory reserves and is being killed.
Michal Hockoa3739662016-07-28 15:45:01 -0700302 * Don't allow any other task to have access to the reserves unless
Michal Hocko862e3072016-10-07 16:58:57 -0700303 * the task has MMF_OOM_SKIP because chances that it would release
Michal Hockoa3739662016-07-28 15:45:01 -0700304 * any memory is quite low.
David Rientjes462607e2012-07-31 16:43:40 -0700305 */
Michal Hocko862e3072016-10-07 16:58:57 -0700306 if (!is_sysrq_oom(oc) && tsk_is_oom_victim(task)) {
307 if (test_bit(MMF_OOM_SKIP, &task->signal->oom_mm->flags))
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700308 goto next;
309 goto abort;
Michal Hockoa3739662016-07-28 15:45:01 -0700310 }
David Rientjes462607e2012-07-31 16:43:40 -0700311
David Rientjese1e12d22012-12-11 16:02:56 -0800312 /*
313 * If task is allocating a lot of memory and has been marked to be
314 * killed first if it triggers an oom, then select it.
315 */
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700316 if (oom_task_origin(task)) {
317 points = ULONG_MAX;
318 goto select;
319 }
David Rientjese1e12d22012-12-11 16:02:56 -0800320
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700321 points = oom_badness(task, NULL, oc->nodemask, oc->totalpages);
322 if (!points || points < oc->chosen_points)
323 goto next;
324
325 /* Prefer thread group leaders for display purposes */
326 if (points == oc->chosen_points && thread_group_leader(oc->chosen))
327 goto next;
328select:
329 if (oc->chosen)
330 put_task_struct(oc->chosen);
331 get_task_struct(task);
332 oc->chosen = task;
333 oc->chosen_points = points;
334next:
335 return 0;
336abort:
337 if (oc->chosen)
338 put_task_struct(oc->chosen);
339 oc->chosen = (void *)-1UL;
340 return 1;
David Rientjes462607e2012-07-31 16:43:40 -0700341}
342
Christoph Lameter9b0f8b02006-02-20 18:27:52 -0800343/*
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700344 * Simple selection loop. We choose the process with the highest number of
345 * 'points'. In case scan was aborted, oc->chosen is set to -1.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 */
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700347static void select_bad_process(struct oom_control *oc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348{
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700349 if (is_memcg_oom(oc))
350 mem_cgroup_scan_tasks(oc->memcg, oom_evaluate_task, oc);
351 else {
352 struct task_struct *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700354 rcu_read_lock();
355 for_each_process(p)
356 if (oom_evaluate_task(p, oc))
357 break;
358 rcu_read_unlock();
Oleg Nesterov1da4db02014-01-21 15:49:58 -0800359 }
Oleg Nesterov972c4ea2006-09-29 02:01:12 -0700360
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700361 oc->chosen_points = oc->chosen_points * 1000 / oc->totalpages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362}
363
364/**
Randy Dunlap1b578df2008-03-19 17:00:42 -0700365 * dump_tasks - dump current memory state of all system tasks
Wanpeng Lidad75572012-06-20 12:53:01 -0700366 * @memcg: current's memory controller, if constrained
David Rientjese85bfd32010-09-22 13:05:10 -0700367 * @nodemask: nodemask passed to page allocator for mempolicy ooms
Randy Dunlap1b578df2008-03-19 17:00:42 -0700368 *
David Rientjese85bfd32010-09-22 13:05:10 -0700369 * Dumps the current memory state of all eligible tasks. Tasks not in the same
370 * memcg, not in the same cpuset, or bound to a disjoint set of mempolicy nodes
371 * are not shown.
David Rientjesde34d962012-07-31 16:42:56 -0700372 * State information includes task's pid, uid, tgid, vm size, rss, nr_ptes,
373 * swapents, oom_score_adj value, and name.
David Rientjesfef1bdd2008-02-07 00:14:07 -0800374 */
Johannes Weiner2314b422014-12-10 15:44:33 -0800375static void dump_tasks(struct mem_cgroup *memcg, const nodemask_t *nodemask)
David Rientjesfef1bdd2008-02-07 00:14:07 -0800376{
KOSAKI Motohiroc55db952010-08-09 17:18:46 -0700377 struct task_struct *p;
378 struct task_struct *task;
David Rientjesfef1bdd2008-02-07 00:14:07 -0800379
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800380 pr_info("[ pid ] uid tgid total_vm rss nr_ptes nr_pmds swapents oom_score_adj name\n");
David Rientjes6b0c81b2012-07-31 16:43:45 -0700381 rcu_read_lock();
KOSAKI Motohiroc55db952010-08-09 17:18:46 -0700382 for_each_process(p) {
Johannes Weiner72835c82012-01-12 17:18:32 -0800383 if (oom_unkillable_task(p, memcg, nodemask))
David Rientjesfef1bdd2008-02-07 00:14:07 -0800384 continue;
385
KOSAKI Motohiroc55db952010-08-09 17:18:46 -0700386 task = find_lock_task_mm(p);
387 if (!task) {
David Rientjes6d2661e2009-05-28 14:34:19 -0700388 /*
David Rientjes74ab7f12010-08-09 17:18:46 -0700389 * This is a kthread or all of p's threads have already
390 * detached their mm's. There's no need to report
KOSAKI Motohiroc55db952010-08-09 17:18:46 -0700391 * them; they can't be oom killed anyway.
David Rientjes6d2661e2009-05-28 14:34:19 -0700392 */
David Rientjes6d2661e2009-05-28 14:34:19 -0700393 continue;
394 }
KOSAKI Motohiroc55db952010-08-09 17:18:46 -0700395
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800396 pr_info("[%5d] %5d %5d %8lu %8lu %7ld %7ld %8lu %5hd %s\n",
Eric W. Biederman078de5f2012-02-08 07:00:08 -0800397 task->pid, from_kuid(&init_user_ns, task_uid(task)),
398 task->tgid, task->mm->total_vm, get_mm_rss(task->mm),
Kirill A. Shutemove1f56c82013-11-14 14:30:48 -0800399 atomic_long_read(&task->mm->nr_ptes),
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800400 mm_nr_pmds(task->mm),
David Rientjesde34d962012-07-31 16:42:56 -0700401 get_mm_counter(task->mm, MM_SWAPENTS),
David Rientjesa63d83f2010-08-09 17:19:46 -0700402 task->signal->oom_score_adj, task->comm);
KOSAKI Motohiroc55db952010-08-09 17:18:46 -0700403 task_unlock(task);
404 }
David Rientjes6b0c81b2012-07-31 16:43:45 -0700405 rcu_read_unlock();
David Rientjesfef1bdd2008-02-07 00:14:07 -0800406}
407
Vladimir Davydov2a966b72016-07-26 15:22:33 -0700408static void dump_header(struct oom_control *oc, struct task_struct *p)
David Rientjes1b604d72009-12-14 17:57:47 -0800409{
David Rientjes299c5172017-02-24 14:55:42 -0800410 pr_warn("%s invoked oom-killer: gfp_mask=%#x(%pGg), nodemask=",
411 current->comm, oc->gfp_mask, &oc->gfp_mask);
412 if (oc->nodemask)
413 pr_cont("%*pbl", nodemask_pr_args(oc->nodemask));
414 else
415 pr_cont("(null)");
416 pr_cont(", order=%d, oom_score_adj=%hd\n",
417 oc->order, current->signal->oom_score_adj);
Michal Hocko92549902016-10-07 16:59:33 -0700418 if (!IS_ENABLED(CONFIG_COMPACTION) && oc->order)
419 pr_warn("COMPACTION is disabled!!!\n");
Vlastimil Babkaa0795cd2016-03-15 14:56:05 -0700420
David Rientjesda39da32015-11-05 18:48:05 -0800421 cpuset_print_current_mems_allowed();
David Rientjes1b604d72009-12-14 17:57:47 -0800422 dump_stack();
Vladimir Davydov2a966b72016-07-26 15:22:33 -0700423 if (oc->memcg)
424 mem_cgroup_print_oom_info(oc->memcg, p);
Sha Zhengju58cf1882013-02-22 16:32:05 -0800425 else
David Rientjes299c5172017-02-24 14:55:42 -0800426 show_mem(SHOW_MEM_FILTER_NODES, oc->nodemask);
David Rientjes1b604d72009-12-14 17:57:47 -0800427 if (sysctl_oom_dump_tasks)
Vladimir Davydov2a966b72016-07-26 15:22:33 -0700428 dump_tasks(oc->memcg, oc->nodemask);
David Rientjes1b604d72009-12-14 17:57:47 -0800429}
430
Michal Hocko5695be12014-10-20 18:12:32 +0200431/*
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800432 * Number of OOM victims in flight
Michal Hocko5695be12014-10-20 18:12:32 +0200433 */
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800434static atomic_t oom_victims = ATOMIC_INIT(0);
435static DECLARE_WAIT_QUEUE_HEAD(oom_victims_wait);
Michal Hocko5695be12014-10-20 18:12:32 +0200436
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700437static bool oom_killer_disabled __read_mostly;
Michal Hocko5695be12014-10-20 18:12:32 +0200438
Michal Hockobc448e82016-03-25 14:20:30 -0700439#define K(x) ((x) << (PAGE_SHIFT-10))
440
Michal Hocko3ef22df2016-05-19 17:13:12 -0700441/*
442 * task->mm can be NULL if the task is the exited group leader. So to
443 * determine whether the task is using a particular mm, we examine all the
444 * task's threads: if one of those is using this mm then this task was also
445 * using it.
446 */
Michal Hocko44a70ade2016-07-28 15:44:43 -0700447bool process_shares_mm(struct task_struct *p, struct mm_struct *mm)
Michal Hocko3ef22df2016-05-19 17:13:12 -0700448{
449 struct task_struct *t;
450
451 for_each_thread(p, t) {
452 struct mm_struct *t_mm = READ_ONCE(t->mm);
453 if (t_mm)
454 return t_mm == mm;
455 }
456 return false;
457}
458
Michal Hockoaac45362016-03-25 14:20:24 -0700459#ifdef CONFIG_MMU
460/*
461 * OOM Reaper kernel thread which tries to reap the memory used by the OOM
462 * victim (if that is possible) to help the OOM killer to move on.
463 */
464static struct task_struct *oom_reaper_th;
Michal Hockoaac45362016-03-25 14:20:24 -0700465static DECLARE_WAIT_QUEUE_HEAD(oom_reaper_wait);
Vladimir Davydov29c696e2016-03-25 14:20:39 -0700466static struct task_struct *oom_reaper_list;
Michal Hocko03049262016-03-25 14:20:33 -0700467static DEFINE_SPINLOCK(oom_reaper_lock);
468
David Rientjes2270dfc2018-05-11 16:02:04 -0700469void __oom_reap_task_mm(struct mm_struct *mm)
Michal Hockoaac45362016-03-25 14:20:24 -0700470{
Michal Hockoaac45362016-03-25 14:20:24 -0700471 struct vm_area_struct *vma;
David Rientjes2270dfc2018-05-11 16:02:04 -0700472
473 /*
474 * Tell all users of get_user/copy_from_user etc... that the content
475 * is no longer stable. No barriers really needed because unmapping
476 * should imply barriers already and the reader would hit a page fault
477 * if it stumbled over a reaped memory.
478 */
479 set_bit(MMF_UNSTABLE, &mm->flags);
480
481 for (vma = mm->mmap ; vma; vma = vma->vm_next) {
482 if (!can_madv_dontneed_vma(vma))
483 continue;
484
485 /*
486 * Only anonymous pages have a good chance to be dropped
487 * without additional steps which we cannot afford as we
488 * are OOM already.
489 *
490 * We do not even care about fs backed pages because all
491 * which are reclaimable have already been reclaimed and
492 * we do not want to block exit_mmap by keeping mm ref
493 * count elevated without a good reason.
494 */
495 if (vma_is_anonymous(vma) || !(vma->vm_flags & VM_SHARED)) {
496 struct mmu_gather tlb;
497
498 tlb_gather_mmu(&tlb, mm, vma->vm_start, vma->vm_end);
499 unmap_page_range(&tlb, vma, vma->vm_start, vma->vm_end,
500 NULL);
501 tlb_finish_mmu(&tlb, vma->vm_start, vma->vm_end);
502 }
503 }
504}
505
506static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm)
507{
Michal Hockoaac45362016-03-25 14:20:24 -0700508 bool ret = true;
509
Michal Hocko36324a92016-03-25 14:20:27 -0700510 /*
Michal Hockoe2fe1452016-05-27 14:27:35 -0700511 * We have to make sure to not race with the victim exit path
512 * and cause premature new oom victim selection:
David Rientjes2270dfc2018-05-11 16:02:04 -0700513 * oom_reap_task_mm exit_mm
Michal Hockoe5e3f4c2016-07-26 15:24:50 -0700514 * mmget_not_zero
Michal Hockoe2fe1452016-05-27 14:27:35 -0700515 * mmput
516 * atomic_dec_and_test
517 * exit_oom_victim
518 * [...]
519 * out_of_memory
520 * select_bad_process
521 * # no TIF_MEMDIE task selects new victim
522 * unmap_page_range # frees some memory
523 */
524 mutex_lock(&oom_lock);
525
Michal Hockoaac45362016-03-25 14:20:24 -0700526 if (!down_read_trylock(&mm->mmap_sem)) {
527 ret = false;
Roman Gushchin422580c2017-07-10 15:49:05 -0700528 trace_skip_task_reaping(tsk->pid);
Tetsuo Handa7ebffa42016-10-07 16:58:45 -0700529 goto unlock_oom;
Michal Hockoe5e3f4c2016-07-26 15:24:50 -0700530 }
531
532 /*
Michal Hocko4d4bbd82017-10-03 16:14:50 -0700533 * If the mm has notifiers then we would need to invalidate them around
534 * unmap_page_range and that is risky because notifiers can sleep and
535 * what they do is basically undeterministic. So let's have a short
536 * sleep to give the oom victim some more time.
537 * TODO: we really want to get rid of this ugly hack and make sure that
538 * notifiers cannot block for unbounded amount of time and add
539 * mmu_notifier_invalidate_range_{start,end} around unmap_page_range
540 */
541 if (mm_has_notifiers(mm)) {
542 up_read(&mm->mmap_sem);
543 schedule_timeout_idle(HZ);
544 goto unlock_oom;
545 }
546
547 /*
Andrea Arcangeli21292582017-09-06 16:25:00 -0700548 * MMF_OOM_SKIP is set by exit_mmap when the OOM reaper can't
549 * work on the mm anymore. The check for MMF_OOM_SKIP must run
550 * under mmap_sem for reading because it serializes against the
551 * down_write();up_write() cycle in exit_mmap().
Michal Hockoe5e3f4c2016-07-26 15:24:50 -0700552 */
Andrea Arcangeli21292582017-09-06 16:25:00 -0700553 if (test_bit(MMF_OOM_SKIP, &mm->flags)) {
Michal Hockoe5e3f4c2016-07-26 15:24:50 -0700554 up_read(&mm->mmap_sem);
Roman Gushchin422580c2017-07-10 15:49:05 -0700555 trace_skip_task_reaping(tsk->pid);
Tetsuo Handa7ebffa42016-10-07 16:58:45 -0700556 goto unlock_oom;
Michal Hockoaac45362016-03-25 14:20:24 -0700557 }
558
Roman Gushchin422580c2017-07-10 15:49:05 -0700559 trace_start_task_reaping(tsk->pid);
560
David Rientjes2270dfc2018-05-11 16:02:04 -0700561 __oom_reap_task_mm(mm);
Michal Hocko3f70dc32016-10-07 16:59:06 -0700562
Michal Hockobc448e82016-03-25 14:20:30 -0700563 pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n",
564 task_pid_nr(tsk), tsk->comm,
565 K(get_mm_counter(mm, MM_ANONPAGES)),
566 K(get_mm_counter(mm, MM_FILEPAGES)),
567 K(get_mm_counter(mm, MM_SHMEMPAGES)));
Michal Hockoaac45362016-03-25 14:20:24 -0700568 up_read(&mm->mmap_sem);
Michal Hocko36324a92016-03-25 14:20:27 -0700569
Roman Gushchin422580c2017-07-10 15:49:05 -0700570 trace_finish_task_reaping(tsk->pid);
Michal Hockoe5e3f4c2016-07-26 15:24:50 -0700571unlock_oom:
572 mutex_unlock(&oom_lock);
Michal Hockoaac45362016-03-25 14:20:24 -0700573 return ret;
574}
575
Michal Hockobc448e82016-03-25 14:20:30 -0700576#define MAX_OOM_REAP_RETRIES 10
Michal Hocko36324a92016-03-25 14:20:27 -0700577static void oom_reap_task(struct task_struct *tsk)
Michal Hockoaac45362016-03-25 14:20:24 -0700578{
579 int attempts = 0;
Michal Hocko26db62f2016-10-07 16:58:51 -0700580 struct mm_struct *mm = tsk->signal->oom_mm;
Michal Hockoaac45362016-03-25 14:20:24 -0700581
582 /* Retry the down_read_trylock(mmap_sem) a few times */
David Rientjes2270dfc2018-05-11 16:02:04 -0700583 while (attempts++ < MAX_OOM_REAP_RETRIES && !oom_reap_task_mm(tsk, mm))
Michal Hockoaac45362016-03-25 14:20:24 -0700584 schedule_timeout_idle(HZ/10);
585
Tetsuo Handa7ebffa42016-10-07 16:58:45 -0700586 if (attempts <= MAX_OOM_REAP_RETRIES)
587 goto done;
Michal Hocko11a410d2016-07-28 15:44:58 -0700588
Tetsuo Handa7ebffa42016-10-07 16:58:45 -0700589 pr_info("oom_reaper: unable to reap pid:%d (%s)\n",
590 task_pid_nr(tsk), tsk->comm);
Tetsuo Handa7ebffa42016-10-07 16:58:45 -0700591 debug_show_all_locks();
Michal Hockobc448e82016-03-25 14:20:30 -0700592
Tetsuo Handa7ebffa42016-10-07 16:58:45 -0700593done:
Michal Hocko449d7772016-05-19 17:13:15 -0700594 tsk->oom_reaper_list = NULL;
Michal Hocko449d7772016-05-19 17:13:15 -0700595
Michal Hocko26db62f2016-10-07 16:58:51 -0700596 /*
597 * Hide this mm from OOM killer because it has been either reaped or
598 * somebody can't call up_write(mmap_sem).
599 */
Michal Hocko862e3072016-10-07 16:58:57 -0700600 set_bit(MMF_OOM_SKIP, &mm->flags);
Michal Hocko26db62f2016-10-07 16:58:51 -0700601
Michal Hockoaac45362016-03-25 14:20:24 -0700602 /* Drop a reference taken by wake_oom_reaper */
Michal Hocko36324a92016-03-25 14:20:27 -0700603 put_task_struct(tsk);
Michal Hockoaac45362016-03-25 14:20:24 -0700604}
605
606static int oom_reaper(void *unused)
607{
608 while (true) {
Michal Hocko03049262016-03-25 14:20:33 -0700609 struct task_struct *tsk = NULL;
Michal Hockoaac45362016-03-25 14:20:24 -0700610
Vladimir Davydov29c696e2016-03-25 14:20:39 -0700611 wait_event_freezable(oom_reaper_wait, oom_reaper_list != NULL);
Michal Hocko03049262016-03-25 14:20:33 -0700612 spin_lock(&oom_reaper_lock);
Vladimir Davydov29c696e2016-03-25 14:20:39 -0700613 if (oom_reaper_list != NULL) {
614 tsk = oom_reaper_list;
615 oom_reaper_list = tsk->oom_reaper_list;
Michal Hocko03049262016-03-25 14:20:33 -0700616 }
617 spin_unlock(&oom_reaper_lock);
618
619 if (tsk)
620 oom_reap_task(tsk);
Michal Hockoaac45362016-03-25 14:20:24 -0700621 }
622
623 return 0;
624}
625
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700626static void wake_oom_reaper(struct task_struct *tsk)
Michal Hockoaac45362016-03-25 14:20:24 -0700627{
Michal Hockoaf8e15c2016-04-01 14:31:34 -0700628 if (!oom_reaper_th)
629 return;
630
Tetsuo Handa73178542019-02-01 14:20:31 -0800631 /* mm is already queued? */
632 if (test_and_set_bit(MMF_OOM_REAP_QUEUED, &tsk->signal->oom_mm->flags))
Michal Hockoaac45362016-03-25 14:20:24 -0700633 return;
634
Michal Hocko36324a92016-03-25 14:20:27 -0700635 get_task_struct(tsk);
Michal Hockoaac45362016-03-25 14:20:24 -0700636
Michal Hocko03049262016-03-25 14:20:33 -0700637 spin_lock(&oom_reaper_lock);
Vladimir Davydov29c696e2016-03-25 14:20:39 -0700638 tsk->oom_reaper_list = oom_reaper_list;
639 oom_reaper_list = tsk;
Michal Hocko03049262016-03-25 14:20:33 -0700640 spin_unlock(&oom_reaper_lock);
Roman Gushchin422580c2017-07-10 15:49:05 -0700641 trace_wake_reaper(tsk->pid);
Michal Hocko03049262016-03-25 14:20:33 -0700642 wake_up(&oom_reaper_wait);
Michal Hockoaac45362016-03-25 14:20:24 -0700643}
644
645static int __init oom_init(void)
646{
647 oom_reaper_th = kthread_run(oom_reaper, NULL, "oom_reaper");
648 if (IS_ERR(oom_reaper_th)) {
649 pr_err("Unable to start OOM reaper %ld. Continuing regardless\n",
650 PTR_ERR(oom_reaper_th));
651 oom_reaper_th = NULL;
652 }
653 return 0;
654}
655subsys_initcall(oom_init)
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700656#else
657static inline void wake_oom_reaper(struct task_struct *tsk)
658{
659}
660#endif /* CONFIG_MMU */
Michal Hockoaac45362016-03-25 14:20:24 -0700661
Michal Hocko49550b62015-02-11 15:26:12 -0800662/**
Johannes Weiner16e95192015-06-24 16:57:07 -0700663 * mark_oom_victim - mark the given task as OOM victim
Michal Hocko49550b62015-02-11 15:26:12 -0800664 * @tsk: task to mark
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800665 *
Johannes Weinerdc564012015-06-24 16:57:19 -0700666 * Has to be called with oom_lock held and never after
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800667 * oom has been disabled already.
Michal Hocko26db62f2016-10-07 16:58:51 -0700668 *
669 * tsk->mm has to be non NULL and caller has to guarantee it is stable (either
670 * under task_lock or operate on the current).
Michal Hocko49550b62015-02-11 15:26:12 -0800671 */
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700672static void mark_oom_victim(struct task_struct *tsk)
Michal Hocko49550b62015-02-11 15:26:12 -0800673{
Michal Hocko26db62f2016-10-07 16:58:51 -0700674 struct mm_struct *mm = tsk->mm;
675
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800676 WARN_ON(oom_killer_disabled);
677 /* OOM killer might race with memcg OOM */
678 if (test_and_set_tsk_thread_flag(tsk, TIF_MEMDIE))
679 return;
Michal Hocko26db62f2016-10-07 16:58:51 -0700680
Michal Hocko26db62f2016-10-07 16:58:51 -0700681 /* oom_mm is bound to the signal struct life time. */
Michal Hocko55fe4692017-12-14 15:33:15 -0800682 if (!cmpxchg(&tsk->signal->oom_mm, NULL, mm)) {
Vegard Nossumf1f10072017-02-27 14:30:07 -0800683 mmgrab(tsk->signal->oom_mm);
Michal Hocko55fe4692017-12-14 15:33:15 -0800684 set_bit(MMF_OOM_VICTIM, &mm->flags);
685 }
Michal Hocko26db62f2016-10-07 16:58:51 -0700686
Michal Hocko63a8ca92015-02-11 15:26:15 -0800687 /*
688 * Make sure that the task is woken up from uninterruptible sleep
689 * if it is frozen because OOM killer wouldn't be able to free
690 * any memory and livelock. freezing_slow_path will tell the freezer
691 * that TIF_MEMDIE tasks should be ignored.
692 */
693 __thaw_task(tsk);
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800694 atomic_inc(&oom_victims);
Roman Gushchin422580c2017-07-10 15:49:05 -0700695 trace_mark_victim(tsk->pid);
Michal Hocko49550b62015-02-11 15:26:12 -0800696}
697
698/**
Johannes Weiner16e95192015-06-24 16:57:07 -0700699 * exit_oom_victim - note the exit of an OOM victim
Michal Hocko49550b62015-02-11 15:26:12 -0800700 */
Tetsuo Handa38531202016-10-07 16:59:03 -0700701void exit_oom_victim(void)
Michal Hocko49550b62015-02-11 15:26:12 -0800702{
Tetsuo Handa38531202016-10-07 16:59:03 -0700703 clear_thread_flag(TIF_MEMDIE);
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800704
Johannes Weinerc38f1022015-06-24 16:57:13 -0700705 if (!atomic_dec_return(&oom_victims))
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800706 wake_up_all(&oom_victims_wait);
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800707}
708
709/**
Michal Hocko7d2e7a22016-10-07 16:59:00 -0700710 * oom_killer_enable - enable OOM killer
711 */
712void oom_killer_enable(void)
713{
714 oom_killer_disabled = false;
Michal Hockod75da002017-05-03 14:54:57 -0700715 pr_info("OOM killer enabled.\n");
Michal Hocko7d2e7a22016-10-07 16:59:00 -0700716}
717
718/**
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800719 * oom_killer_disable - disable OOM killer
Michal Hocko7d2e7a22016-10-07 16:59:00 -0700720 * @timeout: maximum timeout to wait for oom victims in jiffies
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800721 *
722 * Forces all page allocations to fail rather than trigger OOM killer.
Michal Hocko7d2e7a22016-10-07 16:59:00 -0700723 * Will block and wait until all OOM victims are killed or the given
724 * timeout expires.
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800725 *
726 * The function cannot be called when there are runnable user tasks because
727 * the userspace would see unexpected allocation failures as a result. Any
728 * new usage of this function should be consulted with MM people.
729 *
730 * Returns true if successful and false if the OOM killer cannot be
731 * disabled.
732 */
Michal Hocko7d2e7a22016-10-07 16:59:00 -0700733bool oom_killer_disable(signed long timeout)
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800734{
Michal Hocko7d2e7a22016-10-07 16:59:00 -0700735 signed long ret;
736
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800737 /*
Tetsuo Handa6afcf282016-03-17 14:20:45 -0700738 * Make sure to not race with an ongoing OOM killer. Check that the
739 * current is not killed (possibly due to sharing the victim's memory).
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800740 */
Tetsuo Handa6afcf282016-03-17 14:20:45 -0700741 if (mutex_lock_killable(&oom_lock))
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800742 return false;
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800743 oom_killer_disabled = true;
Johannes Weinerdc564012015-06-24 16:57:19 -0700744 mutex_unlock(&oom_lock);
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800745
Michal Hocko7d2e7a22016-10-07 16:59:00 -0700746 ret = wait_event_interruptible_timeout(oom_victims_wait,
747 !atomic_read(&oom_victims), timeout);
748 if (ret <= 0) {
749 oom_killer_enable();
750 return false;
751 }
Michal Hockod75da002017-05-03 14:54:57 -0700752 pr_info("OOM killer disabled.\n");
Michal Hockoc32b3cb2015-02-11 15:26:24 -0800753
754 return true;
755}
756
Michal Hocko1af8bb42016-07-28 15:44:52 -0700757static inline bool __task_will_free_mem(struct task_struct *task)
758{
759 struct signal_struct *sig = task->signal;
760
761 /*
762 * A coredumping process may sleep for an extended period in exit_mm(),
763 * so the oom killer cannot assume that the process will promptly exit
764 * and release memory.
765 */
766 if (sig->flags & SIGNAL_GROUP_COREDUMP)
767 return false;
768
769 if (sig->flags & SIGNAL_GROUP_EXIT)
770 return true;
771
772 if (thread_group_empty(task) && (task->flags & PF_EXITING))
773 return true;
774
775 return false;
776}
777
778/*
779 * Checks whether the given task is dying or exiting and likely to
780 * release its address space. This means that all threads and processes
781 * sharing the same mm have to be killed or exiting.
Michal Hocko091f3622016-07-28 15:45:04 -0700782 * Caller has to make sure that task->mm is stable (hold task_lock or
783 * it operates on the current).
Michal Hocko1af8bb42016-07-28 15:44:52 -0700784 */
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700785static bool task_will_free_mem(struct task_struct *task)
Michal Hocko1af8bb42016-07-28 15:44:52 -0700786{
Michal Hocko091f3622016-07-28 15:45:04 -0700787 struct mm_struct *mm = task->mm;
Michal Hocko1af8bb42016-07-28 15:44:52 -0700788 struct task_struct *p;
Geert Uytterhoevenf33e6f02016-08-11 15:33:09 -0700789 bool ret = true;
Michal Hocko1af8bb42016-07-28 15:44:52 -0700790
Michal Hocko091f3622016-07-28 15:45:04 -0700791 /*
792 * Skip tasks without mm because it might have passed its exit_mm and
793 * exit_oom_victim. oom_reaper could have rescued that but do not rely
794 * on that for now. We can consider find_lock_task_mm in future.
795 */
796 if (!mm)
797 return false;
798
Michal Hocko1af8bb42016-07-28 15:44:52 -0700799 if (!__task_will_free_mem(task))
800 return false;
801
802 /*
Michal Hocko696453e2016-07-28 15:44:55 -0700803 * This task has already been drained by the oom reaper so there are
804 * only small chances it will free some more
805 */
Michal Hocko862e3072016-10-07 16:58:57 -0700806 if (test_bit(MMF_OOM_SKIP, &mm->flags))
Michal Hocko696453e2016-07-28 15:44:55 -0700807 return false;
Michal Hocko696453e2016-07-28 15:44:55 -0700808
Michal Hocko091f3622016-07-28 15:45:04 -0700809 if (atomic_read(&mm->mm_users) <= 1)
Michal Hocko1af8bb42016-07-28 15:44:52 -0700810 return true;
Michal Hocko1af8bb42016-07-28 15:44:52 -0700811
812 /*
Michal Hocko5870c2e2016-10-07 16:57:32 -0700813 * Make sure that all tasks which share the mm with the given tasks
814 * are dying as well to make sure that a) nobody pins its mm and
815 * b) the task is also reapable by the oom reaper.
Michal Hocko1af8bb42016-07-28 15:44:52 -0700816 */
817 rcu_read_lock();
818 for_each_process(p) {
819 if (!process_shares_mm(p, mm))
820 continue;
821 if (same_thread_group(task, p))
822 continue;
823 ret = __task_will_free_mem(p);
824 if (!ret)
825 break;
826 }
827 rcu_read_unlock();
Michal Hocko1af8bb42016-07-28 15:44:52 -0700828
829 return ret;
830}
831
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700832static void oom_kill_process(struct oom_control *oc, const char *message)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833{
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700834 struct task_struct *p = oc->chosen;
835 unsigned int points = oc->chosen_points;
Linus Torvalds52d3c032011-03-14 15:17:07 -0700836 struct task_struct *victim = p;
David Rientjes5e9d8342010-08-09 17:18:51 -0700837 struct task_struct *child;
Oleg Nesterov1da4db02014-01-21 15:49:58 -0800838 struct task_struct *t;
David Rientjes647f2bd2012-03-21 16:33:46 -0700839 struct mm_struct *mm;
Linus Torvalds52d3c032011-03-14 15:17:07 -0700840 unsigned int victim_points = 0;
David Rientjesdc3f21e2012-03-21 16:33:47 -0700841 static DEFINE_RATELIMIT_STATE(oom_rs, DEFAULT_RATELIMIT_INTERVAL,
842 DEFAULT_RATELIMIT_BURST);
Tetsuo Handabb299022016-03-25 14:20:44 -0700843 bool can_oom_reap = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844
Nick Piggin50ec3bb2006-09-25 23:31:29 -0700845 /*
846 * If the task is already exiting, don't alarm the sysadmin or kill
Michal Hockocd04ae12017-09-06 16:24:50 -0700847 * its children or threads, just give it access to memory reserves
848 * so it can die quickly
Nick Piggin50ec3bb2006-09-25 23:31:29 -0700849 */
Michal Hocko091f3622016-07-28 15:45:04 -0700850 task_lock(p);
Michal Hocko1af8bb42016-07-28 15:44:52 -0700851 if (task_will_free_mem(p)) {
Johannes Weiner16e95192015-06-24 16:57:07 -0700852 mark_oom_victim(p);
Michal Hocko1af8bb42016-07-28 15:44:52 -0700853 wake_oom_reaper(p);
Michal Hocko091f3622016-07-28 15:45:04 -0700854 task_unlock(p);
David Rientjes6b0c81b2012-07-31 16:43:45 -0700855 put_task_struct(p);
David Rientjes2a1c9b12012-03-21 16:33:46 -0700856 return;
Nick Piggin50ec3bb2006-09-25 23:31:29 -0700857 }
Michal Hocko091f3622016-07-28 15:45:04 -0700858 task_unlock(p);
Nick Piggin50ec3bb2006-09-25 23:31:29 -0700859
David Rientjesdc3f21e2012-03-21 16:33:47 -0700860 if (__ratelimit(&oom_rs))
Vladimir Davydov2a966b72016-07-26 15:22:33 -0700861 dump_header(oc, p);
David Rientjes8447d952012-03-21 16:33:47 -0700862
Wang Longf0d66472015-06-24 16:58:01 -0700863 pr_err("%s: Kill process %d (%s) score %u or sacrifice child\n",
David Rientjes5e9d8342010-08-09 17:18:51 -0700864 message, task_pid_nr(p), p->comm, points);
Nick Pigginf3af38d2006-12-06 20:31:51 -0800865
David Rientjes5e9d8342010-08-09 17:18:51 -0700866 /*
867 * If any of p's children has a different mm and is eligible for kill,
David Rientjes11239832011-07-25 17:12:17 -0700868 * the one with the highest oom_badness() score is sacrificed for its
David Rientjes5e9d8342010-08-09 17:18:51 -0700869 * parent. This attempts to lose the minimal amount of work done while
870 * still freeing memory.
871 */
David Rientjes6b0c81b2012-07-31 16:43:45 -0700872 read_lock(&tasklist_lock);
Shakeel Butt43f7e8b2019-02-01 14:20:54 -0800873
874 /*
875 * The task 'p' might have already exited before reaching here. The
876 * put_task_struct() will free task_struct 'p' while the loop still try
877 * to access the field of 'p', so, get an extra reference.
878 */
879 get_task_struct(p);
Oleg Nesterov1da4db02014-01-21 15:49:58 -0800880 for_each_thread(p, t) {
David Rientjes5e9d8342010-08-09 17:18:51 -0700881 list_for_each_entry(child, &t->children, sibling) {
David Rientjesa63d83f2010-08-09 17:19:46 -0700882 unsigned int child_points;
David Rientjes5e9d8342010-08-09 17:18:51 -0700883
Oleg Nesterov4d7b3392015-11-05 18:48:26 -0800884 if (process_shares_mm(child, p->mm))
David Rientjesedd45542011-03-22 16:30:12 -0700885 continue;
David Rientjesa63d83f2010-08-09 17:19:46 -0700886 /*
887 * oom_badness() returns 0 if the thread is unkillable
888 */
Vladimir Davydov2a966b72016-07-26 15:22:33 -0700889 child_points = oom_badness(child,
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700890 oc->memcg, oc->nodemask, oc->totalpages);
David Rientjes5e9d8342010-08-09 17:18:51 -0700891 if (child_points > victim_points) {
David Rientjes6b0c81b2012-07-31 16:43:45 -0700892 put_task_struct(victim);
David Rientjes5e9d8342010-08-09 17:18:51 -0700893 victim = child;
894 victim_points = child_points;
David Rientjes6b0c81b2012-07-31 16:43:45 -0700895 get_task_struct(victim);
David Rientjes5e9d8342010-08-09 17:18:51 -0700896 }
Oleg Nesterovdd8e8f42010-08-09 17:18:45 -0700897 }
Oleg Nesterov1da4db02014-01-21 15:49:58 -0800898 }
Shakeel Butt43f7e8b2019-02-01 14:20:54 -0800899 put_task_struct(p);
David Rientjes6b0c81b2012-07-31 16:43:45 -0700900 read_unlock(&tasklist_lock);
Oleg Nesterovdd8e8f42010-08-09 17:18:45 -0700901
David Rientjes6b0c81b2012-07-31 16:43:45 -0700902 p = find_lock_task_mm(victim);
903 if (!p) {
David Rientjes6b0c81b2012-07-31 16:43:45 -0700904 put_task_struct(victim);
David Rientjes647f2bd2012-03-21 16:33:46 -0700905 return;
David Rientjes6b0c81b2012-07-31 16:43:45 -0700906 } else if (victim != p) {
907 get_task_struct(p);
908 put_task_struct(victim);
909 victim = p;
910 }
David Rientjes647f2bd2012-03-21 16:33:46 -0700911
Tetsuo Handa880b7682015-11-05 18:47:51 -0800912 /* Get a reference to safely compare mm after task_unlock(victim) */
David Rientjes647f2bd2012-03-21 16:33:46 -0700913 mm = victim->mm;
Vegard Nossumf1f10072017-02-27 14:30:07 -0800914 mmgrab(mm);
Konstantin Khlebnikov8e675f72017-07-06 15:40:28 -0700915
916 /* Raise event before sending signal: task reaper must see this */
917 count_vm_event(OOM_KILL);
918 count_memcg_event_mm(mm, OOM_KILL);
919
Tetsuo Handa426fb5e2015-11-05 18:47:44 -0800920 /*
Michal Hockocd04ae12017-09-06 16:24:50 -0700921 * We should send SIGKILL before granting access to memory reserves
922 * in order to prevent the OOM victim from depleting the memory
923 * reserves from the user space under its control.
Tetsuo Handa426fb5e2015-11-05 18:47:44 -0800924 */
925 do_send_sig_info(SIGKILL, SEND_SIG_FORCED, victim, true);
Johannes Weiner16e95192015-06-24 16:57:07 -0700926 mark_oom_victim(victim);
Jerome Marchandeca56ff2016-01-14 15:19:26 -0800927 pr_err("Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n",
David Rientjes647f2bd2012-03-21 16:33:46 -0700928 task_pid_nr(victim), victim->comm, K(victim->mm->total_vm),
929 K(get_mm_counter(victim->mm, MM_ANONPAGES)),
Jerome Marchandeca56ff2016-01-14 15:19:26 -0800930 K(get_mm_counter(victim->mm, MM_FILEPAGES)),
931 K(get_mm_counter(victim->mm, MM_SHMEMPAGES)));
David Rientjes647f2bd2012-03-21 16:33:46 -0700932 task_unlock(victim);
933
934 /*
935 * Kill all user processes sharing victim->mm in other thread groups, if
936 * any. They don't get access to memory reserves, though, to avoid
937 * depletion of all memory. This prevents mm->mmap_sem livelock when an
938 * oom killed thread cannot exit because it requires the semaphore and
939 * its contended by another thread trying to allocate memory itself.
940 * That thread will now get access to memory reserves since it has a
941 * pending fatal signal.
942 */
Oleg Nesterov4d4048b2014-01-21 15:50:01 -0800943 rcu_read_lock();
Oleg Nesterovc3190252015-11-05 18:48:23 -0800944 for_each_process(p) {
Oleg Nesterov4d7b3392015-11-05 18:48:26 -0800945 if (!process_shares_mm(p, mm))
Oleg Nesterovc3190252015-11-05 18:48:23 -0800946 continue;
947 if (same_thread_group(p, victim))
948 continue;
Michal Hocko1b51e652016-10-07 16:59:09 -0700949 if (is_global_init(p)) {
Michal Hockoaac45362016-03-25 14:20:24 -0700950 can_oom_reap = false;
Michal Hocko862e3072016-10-07 16:58:57 -0700951 set_bit(MMF_OOM_SKIP, &mm->flags);
Michal Hockoa3739662016-07-28 15:45:01 -0700952 pr_info("oom killer %d (%s) has mm pinned by %d (%s)\n",
953 task_pid_nr(victim), victim->comm,
954 task_pid_nr(p), p->comm);
Oleg Nesterovc3190252015-11-05 18:48:23 -0800955 continue;
Michal Hockoaac45362016-03-25 14:20:24 -0700956 }
Michal Hocko1b51e652016-10-07 16:59:09 -0700957 /*
958 * No use_mm() user needs to read from the userspace so we are
959 * ok to reap it.
960 */
961 if (unlikely(p->flags & PF_KTHREAD))
962 continue;
Oleg Nesterovc3190252015-11-05 18:48:23 -0800963 do_send_sig_info(SIGKILL, SEND_SIG_FORCED, p, true);
964 }
David Rientjes6b0c81b2012-07-31 16:43:45 -0700965 rcu_read_unlock();
David Rientjes647f2bd2012-03-21 16:33:46 -0700966
Michal Hockoaac45362016-03-25 14:20:24 -0700967 if (can_oom_reap)
Michal Hocko36324a92016-03-25 14:20:27 -0700968 wake_oom_reaper(victim);
Michal Hockoaac45362016-03-25 14:20:24 -0700969
Tetsuo Handa880b7682015-11-05 18:47:51 -0800970 mmdrop(mm);
David Rientjes6b0c81b2012-07-31 16:43:45 -0700971 put_task_struct(victim);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972}
David Rientjes647f2bd2012-03-21 16:33:46 -0700973#undef K
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974
David Rientjes309ed882010-08-09 17:18:54 -0700975/*
976 * Determines whether the kernel must panic because of the panic_on_oom sysctl.
977 */
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700978static void check_panic_on_oom(struct oom_control *oc,
979 enum oom_constraint constraint)
David Rientjes309ed882010-08-09 17:18:54 -0700980{
981 if (likely(!sysctl_panic_on_oom))
982 return;
983 if (sysctl_panic_on_oom != 2) {
984 /*
985 * panic_on_oom == 1 only affects CONSTRAINT_NONE, the kernel
986 * does not panic for cpuset, mempolicy, or memcg allocation
987 * failures.
988 */
989 if (constraint != CONSTRAINT_NONE)
990 return;
991 }
David Rientjes071a4be2015-09-08 15:00:42 -0700992 /* Do not panic for oom kills triggered by sysrq */
Yaowei Baidb2a0dd2015-11-06 16:28:06 -0800993 if (is_sysrq_oom(oc))
David Rientjes071a4be2015-09-08 15:00:42 -0700994 return;
Vladimir Davydov2a966b72016-07-26 15:22:33 -0700995 dump_header(oc, NULL);
David Rientjes309ed882010-08-09 17:18:54 -0700996 panic("Out of memory: %s panic_on_oom is enabled\n",
997 sysctl_panic_on_oom == 2 ? "compulsory" : "system-wide");
998}
999
Martin Schwidefsky8bc719d2006-09-25 23:31:20 -07001000static BLOCKING_NOTIFIER_HEAD(oom_notify_list);
1001
1002int register_oom_notifier(struct notifier_block *nb)
1003{
1004 return blocking_notifier_chain_register(&oom_notify_list, nb);
1005}
1006EXPORT_SYMBOL_GPL(register_oom_notifier);
1007
1008int unregister_oom_notifier(struct notifier_block *nb)
1009{
1010 return blocking_notifier_chain_unregister(&oom_notify_list, nb);
1011}
1012EXPORT_SYMBOL_GPL(unregister_oom_notifier);
1013
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014/**
David Rientjes6e0fc462015-09-08 15:00:36 -07001015 * out_of_memory - kill the "best" process when we run out of memory
1016 * @oc: pointer to struct oom_control
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 *
1018 * If we run out of memory, we have the choice between either
1019 * killing a random task (bad), letting the system crash (worse)
1020 * OR try to be smart about which process to kill. Note that we
1021 * don't have to be perfect here, we just have to be good.
1022 */
David Rientjes6e0fc462015-09-08 15:00:36 -07001023bool out_of_memory(struct oom_control *oc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024{
Martin Schwidefsky8bc719d2006-09-25 23:31:20 -07001025 unsigned long freed = 0;
David Rientjese3658932010-08-09 17:18:55 -07001026 enum oom_constraint constraint = CONSTRAINT_NONE;
Martin Schwidefsky8bc719d2006-09-25 23:31:20 -07001027
Johannes Weinerdc564012015-06-24 16:57:19 -07001028 if (oom_killer_disabled)
1029 return false;
1030
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001031 if (!is_memcg_oom(oc)) {
1032 blocking_notifier_call_chain(&oom_notify_list, 0, &freed);
1033 if (freed > 0)
1034 /* Got some memory back in the last second. */
1035 return true;
1036 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037
David Rientjes7b98c2e2010-08-09 17:18:48 -07001038 /*
David Rientjes9ff48682012-12-11 16:01:30 -08001039 * If current has a pending SIGKILL or is exiting, then automatically
1040 * select it. The goal is to allow it to allocate so that it may
1041 * quickly exit and free its memory.
David Rientjes7b98c2e2010-08-09 17:18:48 -07001042 */
Michal Hocko091f3622016-07-28 15:45:04 -07001043 if (task_will_free_mem(current)) {
Johannes Weiner16e95192015-06-24 16:57:07 -07001044 mark_oom_victim(current);
Michal Hocko1af8bb42016-07-28 15:44:52 -07001045 wake_oom_reaper(current);
David Rientjes75e8f8b2015-09-08 15:00:47 -07001046 return true;
David Rientjes7b98c2e2010-08-09 17:18:48 -07001047 }
1048
Christoph Lameter9b0f8b02006-02-20 18:27:52 -08001049 /*
Michal Hocko3da88fb2016-05-19 17:13:09 -07001050 * The OOM killer does not compensate for IO-less reclaim.
1051 * pagefault_out_of_memory lost its gfp context so we have to
1052 * make sure exclude 0 mask - all other users should have at least
Tetsuo Handa7bd49f02019-09-23 15:37:08 -07001053 * ___GFP_DIRECT_RECLAIM to get here. But mem_cgroup_oom() has to
1054 * invoke the OOM killer even if it is a GFP_NOFS allocation.
Michal Hocko3da88fb2016-05-19 17:13:09 -07001055 */
Tetsuo Handa7bd49f02019-09-23 15:37:08 -07001056 if (oc->gfp_mask && !(oc->gfp_mask & __GFP_FS) && !is_memcg_oom(oc))
Michal Hocko3da88fb2016-05-19 17:13:09 -07001057 return true;
1058
1059 /*
Christoph Lameter9b0f8b02006-02-20 18:27:52 -08001060 * Check if there were limitations on the allocation (only relevant for
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001061 * NUMA and memcg) that may require different handling.
Christoph Lameter9b0f8b02006-02-20 18:27:52 -08001062 */
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001063 constraint = constrained_alloc(oc);
David Rientjes6e0fc462015-09-08 15:00:36 -07001064 if (constraint != CONSTRAINT_MEMORY_POLICY)
1065 oc->nodemask = NULL;
Vladimir Davydov2a966b72016-07-26 15:22:33 -07001066 check_panic_on_oom(oc, constraint);
David Rientjes0aad4b32010-08-09 17:18:59 -07001067
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001068 if (!is_memcg_oom(oc) && sysctl_oom_kill_allocating_task &&
1069 current->mm && !oom_unkillable_task(current, NULL, oc->nodemask) &&
David Rientjes121d1ba2012-07-31 16:42:55 -07001070 current->signal->oom_score_adj != OOM_SCORE_ADJ_MIN) {
David Rientjes6b0c81b2012-07-31 16:43:45 -07001071 get_task_struct(current);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001072 oc->chosen = current;
1073 oom_kill_process(oc, "Out of memory (oom_kill_allocating_task)");
David Rientjes75e8f8b2015-09-08 15:00:47 -07001074 return true;
David Rientjes0aad4b32010-08-09 17:18:59 -07001075 }
1076
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001077 select_bad_process(oc);
David Rientjes0aad4b32010-08-09 17:18:59 -07001078 /* Found nothing?!?! Either we hang forever, or we panic. */
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001079 if (!oc->chosen && !is_sysrq_oom(oc) && !is_memcg_oom(oc)) {
Vladimir Davydov2a966b72016-07-26 15:22:33 -07001080 dump_header(oc, NULL);
David Rientjes0aad4b32010-08-09 17:18:59 -07001081 panic("Out of memory and no killable processes...\n");
1082 }
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001083 if (oc->chosen && oc->chosen != (void *)-1UL) {
1084 oom_kill_process(oc, !is_memcg_oom(oc) ? "Out of memory" :
1085 "Memory cgroup out of memory");
David Rientjes75e8f8b2015-09-08 15:00:47 -07001086 /*
1087 * Give the killed process a good chance to exit before trying
1088 * to allocate memory again.
1089 */
David Rientjes4f774b92012-07-31 16:42:37 -07001090 schedule_timeout_killable(1);
David Rientjes75e8f8b2015-09-08 15:00:47 -07001091 }
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001092 return !!oc->chosen;
Michal Hockoc32b3cb2015-02-11 15:26:24 -08001093}
1094
David Rientjese3658932010-08-09 17:18:55 -07001095/*
1096 * The pagefault handler calls here because it is out of memory, so kill a
Vladimir Davydov798fd752016-07-26 15:22:30 -07001097 * memory-hogging task. If oom_lock is held by somebody else, a parallel oom
1098 * killing is already in progress so do nothing.
David Rientjese3658932010-08-09 17:18:55 -07001099 */
1100void pagefault_out_of_memory(void)
1101{
David Rientjes6e0fc462015-09-08 15:00:36 -07001102 struct oom_control oc = {
1103 .zonelist = NULL,
1104 .nodemask = NULL,
Vladimir Davydov2a966b72016-07-26 15:22:33 -07001105 .memcg = NULL,
David Rientjes6e0fc462015-09-08 15:00:36 -07001106 .gfp_mask = 0,
1107 .order = 0,
David Rientjes6e0fc462015-09-08 15:00:36 -07001108 };
1109
Johannes Weiner49426422013-10-16 13:46:59 -07001110 if (mem_cgroup_oom_synchronize(true))
Johannes Weinerdc564012015-06-24 16:57:19 -07001111 return;
Johannes Weiner3812c8c2013-09-12 15:13:44 -07001112
Johannes Weinerdc564012015-06-24 16:57:19 -07001113 if (!mutex_trylock(&oom_lock))
1114 return;
Tetsuo Handaa1048082016-10-07 17:00:49 -07001115 out_of_memory(&oc);
Johannes Weinerdc564012015-06-24 16:57:19 -07001116 mutex_unlock(&oom_lock);
David Rientjese3658932010-08-09 17:18:55 -07001117}