blob: 70ab5d664332694e92305331f13ed15a35ab1956 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1995 - 2000 by Ralf Baechle
7 */
Ralf Baechlec3fc5cd2013-05-29 01:07:19 +02008#include <linux/context_tracking.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/signal.h>
10#include <linux/sched.h>
11#include <linux/interrupt.h>
12#include <linux/kernel.h>
13#include <linux/errno.h>
14#include <linux/string.h>
15#include <linux/types.h>
16#include <linux/ptrace.h>
17#include <linux/mman.h>
18#include <linux/mm.h>
19#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/module.h>
David Daneyc1bf2072010-08-03 11:22:20 -070021#include <linux/kprobes.h>
Deng-Cheng Zhu7f788d22010-10-12 19:37:21 +080022#include <linux/perf_event.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
24#include <asm/branch.h>
25#include <asm/mmu_context.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <asm/uaccess.h>
27#include <asm/ptrace.h>
Thiemo Seufer16033d62005-02-19 13:56:04 +000028#include <asm/highmem.h> /* For VMALLOC_END */
David Daneyc1bf2072010-08-03 11:22:20 -070029#include <linux/kdebug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
31/*
32 * This routine handles page faults. It determines the address,
33 * and the problem, and then passes it off to one of the appropriate
34 * routines.
35 */
Ralf Baechlec3fc5cd2013-05-29 01:07:19 +020036static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
37 unsigned long address)
Linus Torvalds1da177e2005-04-16 15:20:36 -070038{
39 struct vm_area_struct * vma = NULL;
40 struct task_struct *tsk = current;
41 struct mm_struct *mm = tsk->mm;
42 const int field = sizeof(unsigned long) * 2;
43 siginfo_t info;
Nick Piggin83c54072007-07-19 01:47:05 -070044 int fault;
Johannes Weiner759496b2013-09-12 15:13:39 -070045 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
47#if 0
Ralf Baechled6f70362007-03-29 22:30:01 +010048 printk("Cpu%d[%s:%d:%0*lx:%ld:%0*lx]\n", raw_smp_processor_id(),
Linus Torvalds1da177e2005-04-16 15:20:36 -070049 current->comm, current->pid, field, address, write,
50 field, regs->cp0_epc);
51#endif
52
David Daneyc1bf2072010-08-03 11:22:20 -070053#ifdef CONFIG_KPROBES
54 /*
Ralf Baechle70342282013-01-22 12:59:30 +010055 * This is to notify the fault handler of the kprobes. The
David Daneyc1bf2072010-08-03 11:22:20 -070056 * exception code is redundant as it is also carried in REGS,
57 * but we pass it anyhow.
58 */
59 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1,
60 (regs->cp0_cause >> 2) & 0x1f, SIGSEGV) == NOTIFY_STOP)
61 return;
62#endif
63
Linus Torvalds1da177e2005-04-16 15:20:36 -070064 info.si_code = SEGV_MAPERR;
65
66 /*
67 * We fault-in kernel-space virtual memory on-demand. The
68 * 'reference' page table is init_mm.pgd.
69 *
70 * NOTE! We MUST NOT take any locks for this case. We may
71 * be in an interrupt or a critical region, and should
72 * only copy the information from the master page table,
73 * nothing more.
74 */
David Daney2ca2ebf2009-09-02 15:47:34 -070075#ifdef CONFIG_64BIT
76# define VMALLOC_FAULT_TARGET no_context
77#else
78# define VMALLOC_FAULT_TARGET vmalloc_fault
79#endif
80
Thiemo Seufer16033d62005-02-19 13:56:04 +000081 if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END))
David Daney2ca2ebf2009-09-02 15:47:34 -070082 goto VMALLOC_FAULT_TARGET;
Atsushi Nemoto656be922006-10-26 00:08:31 +090083#ifdef MODULE_START
84 if (unlikely(address >= MODULE_START && address < MODULE_END))
David Daney2ca2ebf2009-09-02 15:47:34 -070085 goto VMALLOC_FAULT_TARGET;
Atsushi Nemoto656be922006-10-26 00:08:31 +090086#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
88 /*
89 * If we're in an interrupt or have no user
90 * context, we must not take the fault..
91 */
92 if (in_atomic() || !mm)
93 goto bad_area_nosemaphore;
94
Johannes Weiner759496b2013-09-12 15:13:39 -070095 if (user_mode(regs))
96 flags |= FAULT_FLAG_USER;
Kautuk Consul43ca4952011-12-23 16:52:42 +053097retry:
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 down_read(&mm->mmap_sem);
99 vma = find_vma(mm, address);
100 if (!vma)
101 goto bad_area;
102 if (vma->vm_start <= address)
103 goto good_area;
104 if (!(vma->vm_flags & VM_GROWSDOWN))
105 goto bad_area;
106 if (expand_stack(vma, address))
107 goto bad_area;
108/*
109 * Ok, we have a good vm_area for this memory access, so
110 * we can handle it..
111 */
112good_area:
113 info.si_code = SEGV_ACCERR;
114
115 if (write) {
116 if (!(vma->vm_flags & VM_WRITE))
117 goto bad_area;
Johannes Weiner759496b2013-09-12 15:13:39 -0700118 flags |= FAULT_FLAG_WRITE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119 } else {
Steven J. Hill05857c62012-09-13 16:51:46 -0500120 if (cpu_has_rixi) {
David Daney6dd93442010-02-10 15:12:47 -0800121 if (address == regs->cp0_epc && !(vma->vm_flags & VM_EXEC)) {
122#if 0
123 pr_notice("Cpu%d[%s:%d:%0*lx:%ld:%0*lx] XI violation\n",
124 raw_smp_processor_id(),
125 current->comm, current->pid,
126 field, address, write,
127 field, regs->cp0_epc);
128#endif
129 goto bad_area;
130 }
131 if (!(vma->vm_flags & VM_READ)) {
132#if 0
133 pr_notice("Cpu%d[%s:%d:%0*lx:%ld:%0*lx] RI violation\n",
134 raw_smp_processor_id(),
135 current->comm, current->pid,
136 field, address, write,
137 field, regs->cp0_epc);
138#endif
139 goto bad_area;
140 }
141 } else {
142 if (!(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)))
143 goto bad_area;
144 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145 }
146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 /*
148 * If for any reason at all we couldn't handle the fault,
149 * make sure we exit gracefully rather than endlessly redo
150 * the fault.
151 */
Kautuk Consul43ca4952011-12-23 16:52:42 +0530152 fault = handle_mm_fault(mm, vma, address, flags);
153
154 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
155 return;
156
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200157 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
Nick Piggin83c54072007-07-19 01:47:05 -0700158 if (unlikely(fault & VM_FAULT_ERROR)) {
159 if (fault & VM_FAULT_OOM)
160 goto out_of_memory;
Linus Torvalds33692f22015-01-29 10:51:32 -0800161 else if (fault & VM_FAULT_SIGSEGV)
162 goto bad_area;
Nick Piggin83c54072007-07-19 01:47:05 -0700163 else if (fault & VM_FAULT_SIGBUS)
164 goto do_sigbus;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 BUG();
166 }
Kautuk Consul43ca4952011-12-23 16:52:42 +0530167 if (flags & FAULT_FLAG_ALLOW_RETRY) {
168 if (fault & VM_FAULT_MAJOR) {
169 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
170 regs, address);
171 tsk->maj_flt++;
172 } else {
173 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
174 regs, address);
175 tsk->min_flt++;
176 }
177 if (fault & VM_FAULT_RETRY) {
178 flags &= ~FAULT_FLAG_ALLOW_RETRY;
Shaohua Li45cac652012-10-08 16:32:19 -0700179 flags |= FAULT_FLAG_TRIED;
Kautuk Consul43ca4952011-12-23 16:52:42 +0530180
181 /*
182 * No need to up_read(&mm->mmap_sem) as we would
183 * have already released it in __lock_page_or_retry
184 * in mm/filemap.c.
185 */
186
187 goto retry;
188 }
Deng-Cheng Zhu7f788d22010-10-12 19:37:21 +0800189 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190
191 up_read(&mm->mmap_sem);
192 return;
193
194/*
195 * Something tried to access memory that isn't in our memory map..
196 * Fix it, but check if it's kernel or user first..
197 */
198bad_area:
199 up_read(&mm->mmap_sem);
200
201bad_area_nosemaphore:
202 /* User mode accesses just cause a SIGSEGV */
203 if (user_mode(regs)) {
204 tsk->thread.cp0_badvaddr = address;
205 tsk->thread.error_code = write;
206#if 0
207 printk("do_page_fault() #2: sending SIGSEGV to %s for "
208 "invalid %s\n%0*lx (epc == %0*lx, ra == %0*lx)\n",
209 tsk->comm,
210 write ? "write access to" : "read access from",
211 field, address,
212 field, (unsigned long) regs->cp0_epc,
213 field, (unsigned long) regs->regs[31]);
214#endif
215 info.si_signo = SIGSEGV;
216 info.si_errno = 0;
217 /* info.si_code has been set above */
Ralf Baechlefe00f942005-03-01 19:22:29 +0000218 info.si_addr = (void __user *) address;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219 force_sig_info(SIGSEGV, &info, tsk);
220 return;
221 }
222
223no_context:
Ralf Baechle70342282013-01-22 12:59:30 +0100224 /* Are we prepared to handle this kernel fault? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 if (fixup_exception(regs)) {
226 current->thread.cp0_baduaddr = address;
227 return;
228 }
229
230 /*
231 * Oops. The kernel tried to access some bad page. We'll have to
232 * terminate things with extreme prejudice.
233 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 bust_spinlocks(1);
235
236 printk(KERN_ALERT "CPU %d Unable to handle kernel paging request at "
237 "virtual address %0*lx, epc == %0*lx, ra == %0*lx\n",
Ralf Baechled6f70362007-03-29 22:30:01 +0100238 raw_smp_processor_id(), field, address, field, regs->cp0_epc,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 field, regs->regs[31]);
240 die("Oops", regs);
241
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242out_of_memory:
Ralf Baechlec7c1e382009-01-12 00:09:13 +0000243 /*
244 * We ran out of memory, call the OOM killer, and return the userspace
245 * (which will retry the fault, or kill us if we got oom-killed).
246 */
Akinobu Mitaa887b4d2009-07-04 01:33:09 +0900247 up_read(&mm->mmap_sem);
Johannes Weiner87134102013-09-12 15:13:38 -0700248 if (!user_mode(regs))
249 goto no_context;
Ralf Baechlec7c1e382009-01-12 00:09:13 +0000250 pagefault_out_of_memory();
251 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252
253do_sigbus:
254 up_read(&mm->mmap_sem);
255
256 /* Kernel mode? Handle exceptions or die */
257 if (!user_mode(regs))
258 goto no_context;
Ralf Baechle41c594a2006-04-05 09:45:45 +0100259 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 /*
261 * Send a sigbus, regardless of whether we were in kernel
262 * or user mode.
263 */
Ralf Baechle41c594a2006-04-05 09:45:45 +0100264#if 0
265 printk("do_page_fault() #3: sending SIGBUS to %s for "
266 "invalid %s\n%0*lx (epc == %0*lx, ra == %0*lx)\n",
267 tsk->comm,
268 write ? "write access to" : "read access from",
269 field, address,
270 field, (unsigned long) regs->cp0_epc,
271 field, (unsigned long) regs->regs[31]);
272#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273 tsk->thread.cp0_badvaddr = address;
274 info.si_signo = SIGBUS;
275 info.si_errno = 0;
276 info.si_code = BUS_ADRERR;
Ralf Baechlefe00f942005-03-01 19:22:29 +0000277 info.si_addr = (void __user *) address;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 force_sig_info(SIGBUS, &info, tsk);
279
280 return;
David Daney2ca2ebf2009-09-02 15:47:34 -0700281#ifndef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282vmalloc_fault:
283 {
284 /*
285 * Synchronize this task's top level page-table
286 * with the 'reference' page table.
287 *
288 * Do _not_ use "tsk" here. We might be inside
289 * an interrupt in the middle of a task switch..
290 */
291 int offset = __pgd_offset(address);
292 pgd_t *pgd, *pgd_k;
Ralf Baechlec6e8b582005-02-10 12:19:59 +0000293 pud_t *pud, *pud_k;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 pmd_t *pmd, *pmd_k;
295 pte_t *pte_k;
296
Ralf Baechled6f70362007-03-29 22:30:01 +0100297 pgd = (pgd_t *) pgd_current[raw_smp_processor_id()] + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 pgd_k = init_mm.pgd + offset;
299
300 if (!pgd_present(*pgd_k))
301 goto no_context;
302 set_pgd(pgd, *pgd_k);
303
Ralf Baechlec6e8b582005-02-10 12:19:59 +0000304 pud = pud_offset(pgd, address);
305 pud_k = pud_offset(pgd_k, address);
306 if (!pud_present(*pud_k))
307 goto no_context;
308
309 pmd = pmd_offset(pud, address);
310 pmd_k = pmd_offset(pud_k, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 if (!pmd_present(*pmd_k))
312 goto no_context;
313 set_pmd(pmd, *pmd_k);
314
315 pte_k = pte_offset_kernel(pmd_k, address);
316 if (!pte_present(*pte_k))
317 goto no_context;
318 return;
319 }
David Daney2ca2ebf2009-09-02 15:47:34 -0700320#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321}
Ralf Baechlec3fc5cd2013-05-29 01:07:19 +0200322
323asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
324 unsigned long write, unsigned long address)
325{
326 enum ctx_state prev_state;
327
328 prev_state = exception_enter();
329 __do_page_fault(regs, write, address);
330 exception_exit(prev_state);
331}