blob: 154ffb489b93de01631ed4810b5042ba6fa8f123 [file] [log] [blame]
Dave Young2965faa2015-09-09 15:38:55 -07001/*
2 * kexec.c - kexec system call core code.
3 * Copyright (C) 2002-2004 Eric Biederman <ebiederm@xmission.com>
4 *
5 * This source code is licensed under the GNU General Public License,
6 * Version 2. See the file COPYING for more details.
7 */
8
Minfei Huangde90a6b2015-11-06 16:32:45 -08009#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
Dave Young2965faa2015-09-09 15:38:55 -070010
11#include <linux/capability.h>
12#include <linux/mm.h>
13#include <linux/file.h>
14#include <linux/slab.h>
15#include <linux/fs.h>
16#include <linux/kexec.h>
17#include <linux/mutex.h>
18#include <linux/list.h>
19#include <linux/highmem.h>
20#include <linux/syscalls.h>
21#include <linux/reboot.h>
22#include <linux/ioport.h>
23#include <linux/hardirq.h>
24#include <linux/elf.h>
25#include <linux/elfcore.h>
26#include <linux/utsname.h>
27#include <linux/numa.h>
28#include <linux/suspend.h>
29#include <linux/device.h>
30#include <linux/freezer.h>
31#include <linux/pm.h>
32#include <linux/cpu.h>
33#include <linux/uaccess.h>
34#include <linux/io.h>
35#include <linux/console.h>
36#include <linux/vmalloc.h>
37#include <linux/swap.h>
38#include <linux/syscore_ops.h>
39#include <linux/compiler.h>
40#include <linux/hugetlb.h>
Josh Poimboeufc207aee2017-06-28 10:11:06 -050041#include <linux/frame.h>
Dave Young2965faa2015-09-09 15:38:55 -070042
43#include <asm/page.h>
44#include <asm/sections.h>
45
46#include <crypto/hash.h>
47#include <crypto/sha.h>
48#include "kexec_internal.h"
49
50DEFINE_MUTEX(kexec_mutex);
51
52/* Per cpu memory for storing cpu states in case of system crash. */
53note_buf_t __percpu *crash_notes;
54
Dave Young2965faa2015-09-09 15:38:55 -070055/* Flag to indicate we are going to kexec a new kernel */
56bool kexec_in_progress = false;
57
58
59/* Location of the reserved area for the crash kernel */
60struct resource crashk_res = {
61 .name = "Crash kernel",
62 .start = 0,
63 .end = 0,
Toshi Kani1a085d02016-01-26 21:57:23 +010064 .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM,
65 .desc = IORES_DESC_CRASH_KERNEL
Dave Young2965faa2015-09-09 15:38:55 -070066};
67struct resource crashk_low_res = {
68 .name = "Crash kernel",
69 .start = 0,
70 .end = 0,
Toshi Kani1a085d02016-01-26 21:57:23 +010071 .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM,
72 .desc = IORES_DESC_CRASH_KERNEL
Dave Young2965faa2015-09-09 15:38:55 -070073};
74
75int kexec_should_crash(struct task_struct *p)
76{
77 /*
78 * If crash_kexec_post_notifiers is enabled, don't run
79 * crash_kexec() here yet, which must be run after panic
80 * notifiers in panic().
81 */
82 if (crash_kexec_post_notifiers)
83 return 0;
84 /*
85 * There are 4 panic() calls in do_exit() path, each of which
86 * corresponds to each of these 4 conditions.
87 */
88 if (in_interrupt() || !p->pid || is_global_init(p) || panic_on_oops)
89 return 1;
90 return 0;
91}
92
Petr Tesarik21db79e2016-08-02 14:06:16 -070093int kexec_crash_loaded(void)
94{
95 return !!kexec_crash_image;
96}
97EXPORT_SYMBOL_GPL(kexec_crash_loaded);
98
Dave Young2965faa2015-09-09 15:38:55 -070099/*
100 * When kexec transitions to the new kernel there is a one-to-one
101 * mapping between physical and virtual addresses. On processors
102 * where you can disable the MMU this is trivial, and easy. For
103 * others it is still a simple predictable page table to setup.
104 *
105 * In that environment kexec copies the new kernel to its final
106 * resting place. This means I can only support memory whose
107 * physical address can fit in an unsigned long. In particular
108 * addresses where (pfn << PAGE_SHIFT) > ULONG_MAX cannot be handled.
109 * If the assembly stub has more restrictive requirements
110 * KEXEC_SOURCE_MEMORY_LIMIT and KEXEC_DEST_MEMORY_LIMIT can be
111 * defined more restrictively in <asm/kexec.h>.
112 *
113 * The code for the transition from the current kernel to the
114 * the new kernel is placed in the control_code_buffer, whose size
115 * is given by KEXEC_CONTROL_PAGE_SIZE. In the best case only a single
116 * page of memory is necessary, but some architectures require more.
117 * Because this memory must be identity mapped in the transition from
118 * virtual to physical addresses it must live in the range
119 * 0 - TASK_SIZE, as only the user space mappings are arbitrarily
120 * modifiable.
121 *
122 * The assembly stub in the control code buffer is passed a linked list
123 * of descriptor pages detailing the source pages of the new kernel,
124 * and the destination addresses of those source pages. As this data
125 * structure is not used in the context of the current OS, it must
126 * be self-contained.
127 *
128 * The code has been made to work with highmem pages and will use a
129 * destination page in its final resting place (if it happens
130 * to allocate it). The end product of this is that most of the
131 * physical address space, and most of RAM can be used.
132 *
133 * Future directions include:
134 * - allocating a page table with the control code buffer identity
135 * mapped, to simplify machine_kexec and make kexec_on_panic more
136 * reliable.
137 */
138
139/*
140 * KIMAGE_NO_DEST is an impossible destination address..., for
141 * allocating pages whose destination address we do not care about.
142 */
143#define KIMAGE_NO_DEST (-1UL)
zhong jiang1730f142016-08-02 14:06:22 -0700144#define PAGE_COUNT(x) (((x) + PAGE_SIZE - 1) >> PAGE_SHIFT)
Dave Young2965faa2015-09-09 15:38:55 -0700145
146static struct page *kimage_alloc_page(struct kimage *image,
147 gfp_t gfp_mask,
148 unsigned long dest);
149
150int sanity_check_segment_list(struct kimage *image)
151{
Minfei Huang4caf9612016-08-02 14:05:45 -0700152 int i;
Dave Young2965faa2015-09-09 15:38:55 -0700153 unsigned long nr_segments = image->nr_segments;
zhong jiang1730f142016-08-02 14:06:22 -0700154 unsigned long total_pages = 0;
Dave Young2965faa2015-09-09 15:38:55 -0700155
156 /*
157 * Verify we have good destination addresses. The caller is
158 * responsible for making certain we don't attempt to load
159 * the new image into invalid or reserved areas of RAM. This
160 * just verifies it is an address we can use.
161 *
162 * Since the kernel does everything in page size chunks ensure
163 * the destination addresses are page aligned. Too many
164 * special cases crop of when we don't do this. The most
165 * insidious is getting overlapping destination addresses
166 * simply because addresses are changed to page size
167 * granularity.
168 */
Dave Young2965faa2015-09-09 15:38:55 -0700169 for (i = 0; i < nr_segments; i++) {
170 unsigned long mstart, mend;
171
172 mstart = image->segment[i].mem;
173 mend = mstart + image->segment[i].memsz;
Russell King465d3772016-08-02 14:05:57 -0700174 if (mstart > mend)
175 return -EADDRNOTAVAIL;
Dave Young2965faa2015-09-09 15:38:55 -0700176 if ((mstart & ~PAGE_MASK) || (mend & ~PAGE_MASK))
Minfei Huang4caf9612016-08-02 14:05:45 -0700177 return -EADDRNOTAVAIL;
Dave Young2965faa2015-09-09 15:38:55 -0700178 if (mend >= KEXEC_DESTINATION_MEMORY_LIMIT)
Minfei Huang4caf9612016-08-02 14:05:45 -0700179 return -EADDRNOTAVAIL;
Dave Young2965faa2015-09-09 15:38:55 -0700180 }
181
182 /* Verify our destination addresses do not overlap.
183 * If we alloed overlapping destination addresses
184 * through very weird things can happen with no
185 * easy explanation as one segment stops on another.
186 */
Dave Young2965faa2015-09-09 15:38:55 -0700187 for (i = 0; i < nr_segments; i++) {
188 unsigned long mstart, mend;
189 unsigned long j;
190
191 mstart = image->segment[i].mem;
192 mend = mstart + image->segment[i].memsz;
193 for (j = 0; j < i; j++) {
194 unsigned long pstart, pend;
195
196 pstart = image->segment[j].mem;
197 pend = pstart + image->segment[j].memsz;
198 /* Do the segments overlap ? */
199 if ((mend > pstart) && (mstart < pend))
Minfei Huang4caf9612016-08-02 14:05:45 -0700200 return -EINVAL;
Dave Young2965faa2015-09-09 15:38:55 -0700201 }
202 }
203
204 /* Ensure our buffer sizes are strictly less than
205 * our memory sizes. This should always be the case,
206 * and it is easier to check up front than to be surprised
207 * later on.
208 */
Dave Young2965faa2015-09-09 15:38:55 -0700209 for (i = 0; i < nr_segments; i++) {
210 if (image->segment[i].bufsz > image->segment[i].memsz)
Minfei Huang4caf9612016-08-02 14:05:45 -0700211 return -EINVAL;
Dave Young2965faa2015-09-09 15:38:55 -0700212 }
213
214 /*
zhong jiang1730f142016-08-02 14:06:22 -0700215 * Verify that no more than half of memory will be consumed. If the
216 * request from userspace is too large, a large amount of time will be
217 * wasted allocating pages, which can cause a soft lockup.
218 */
219 for (i = 0; i < nr_segments; i++) {
220 if (PAGE_COUNT(image->segment[i].memsz) > totalram_pages / 2)
221 return -EINVAL;
222
223 total_pages += PAGE_COUNT(image->segment[i].memsz);
224 }
225
226 if (total_pages > totalram_pages / 2)
227 return -EINVAL;
228
229 /*
Dave Young2965faa2015-09-09 15:38:55 -0700230 * Verify we have good destination addresses. Normally
231 * the caller is responsible for making certain we don't
232 * attempt to load the new image into invalid or reserved
233 * areas of RAM. But crash kernels are preloaded into a
234 * reserved area of ram. We must ensure the addresses
235 * are in the reserved area otherwise preloading the
236 * kernel could corrupt things.
237 */
238
239 if (image->type == KEXEC_TYPE_CRASH) {
Dave Young2965faa2015-09-09 15:38:55 -0700240 for (i = 0; i < nr_segments; i++) {
241 unsigned long mstart, mend;
242
243 mstart = image->segment[i].mem;
244 mend = mstart + image->segment[i].memsz - 1;
245 /* Ensure we are within the crash kernel limits */
Russell King43546d82016-08-02 14:06:04 -0700246 if ((mstart < phys_to_boot_phys(crashk_res.start)) ||
247 (mend > phys_to_boot_phys(crashk_res.end)))
Minfei Huang4caf9612016-08-02 14:05:45 -0700248 return -EADDRNOTAVAIL;
Dave Young2965faa2015-09-09 15:38:55 -0700249 }
250 }
251
252 return 0;
253}
254
255struct kimage *do_kimage_alloc_init(void)
256{
257 struct kimage *image;
258
259 /* Allocate a controlling structure */
260 image = kzalloc(sizeof(*image), GFP_KERNEL);
261 if (!image)
262 return NULL;
263
264 image->head = 0;
265 image->entry = &image->head;
266 image->last_entry = &image->head;
267 image->control_page = ~0; /* By default this does not apply */
268 image->type = KEXEC_TYPE_DEFAULT;
269
270 /* Initialize the list of control pages */
271 INIT_LIST_HEAD(&image->control_pages);
272
273 /* Initialize the list of destination pages */
274 INIT_LIST_HEAD(&image->dest_pages);
275
276 /* Initialize the list of unusable pages */
277 INIT_LIST_HEAD(&image->unusable_pages);
278
279 return image;
280}
281
282int kimage_is_destination_range(struct kimage *image,
283 unsigned long start,
284 unsigned long end)
285{
286 unsigned long i;
287
288 for (i = 0; i < image->nr_segments; i++) {
289 unsigned long mstart, mend;
290
291 mstart = image->segment[i].mem;
292 mend = mstart + image->segment[i].memsz;
293 if ((end > mstart) && (start < mend))
294 return 1;
295 }
296
297 return 0;
298}
299
300static struct page *kimage_alloc_pages(gfp_t gfp_mask, unsigned int order)
301{
302 struct page *pages;
303
304 pages = alloc_pages(gfp_mask, order);
305 if (pages) {
306 unsigned int count, i;
307
308 pages->mapping = NULL;
309 set_page_private(pages, order);
310 count = 1 << order;
311 for (i = 0; i < count; i++)
312 SetPageReserved(pages + i);
313 }
314
315 return pages;
316}
317
318static void kimage_free_pages(struct page *page)
319{
320 unsigned int order, count, i;
321
322 order = page_private(page);
323 count = 1 << order;
324 for (i = 0; i < count; i++)
325 ClearPageReserved(page + i);
326 __free_pages(page, order);
327}
328
329void kimage_free_page_list(struct list_head *list)
330{
Geliang Tang2b246922016-01-20 15:00:34 -0800331 struct page *page, *next;
Dave Young2965faa2015-09-09 15:38:55 -0700332
Geliang Tang2b246922016-01-20 15:00:34 -0800333 list_for_each_entry_safe(page, next, list, lru) {
Dave Young2965faa2015-09-09 15:38:55 -0700334 list_del(&page->lru);
335 kimage_free_pages(page);
336 }
337}
338
339static struct page *kimage_alloc_normal_control_pages(struct kimage *image,
340 unsigned int order)
341{
342 /* Control pages are special, they are the intermediaries
343 * that are needed while we copy the rest of the pages
344 * to their final resting place. As such they must
345 * not conflict with either the destination addresses
346 * or memory the kernel is already using.
347 *
348 * The only case where we really need more than one of
349 * these are for architectures where we cannot disable
350 * the MMU and must instead generate an identity mapped
351 * page table for all of the memory.
352 *
353 * At worst this runs in O(N) of the image size.
354 */
355 struct list_head extra_pages;
356 struct page *pages;
357 unsigned int count;
358
359 count = 1 << order;
360 INIT_LIST_HEAD(&extra_pages);
361
362 /* Loop while I can allocate a page and the page allocated
363 * is a destination page.
364 */
365 do {
366 unsigned long pfn, epfn, addr, eaddr;
367
368 pages = kimage_alloc_pages(KEXEC_CONTROL_MEMORY_GFP, order);
369 if (!pages)
370 break;
Russell King43546d82016-08-02 14:06:04 -0700371 pfn = page_to_boot_pfn(pages);
Dave Young2965faa2015-09-09 15:38:55 -0700372 epfn = pfn + count;
373 addr = pfn << PAGE_SHIFT;
374 eaddr = epfn << PAGE_SHIFT;
375 if ((epfn >= (KEXEC_CONTROL_MEMORY_LIMIT >> PAGE_SHIFT)) ||
376 kimage_is_destination_range(image, addr, eaddr)) {
377 list_add(&pages->lru, &extra_pages);
378 pages = NULL;
379 }
380 } while (!pages);
381
382 if (pages) {
383 /* Remember the allocated page... */
384 list_add(&pages->lru, &image->control_pages);
385
386 /* Because the page is already in it's destination
387 * location we will never allocate another page at
388 * that address. Therefore kimage_alloc_pages
389 * will not return it (again) and we don't need
390 * to give it an entry in image->segment[].
391 */
392 }
393 /* Deal with the destination pages I have inadvertently allocated.
394 *
395 * Ideally I would convert multi-page allocations into single
396 * page allocations, and add everything to image->dest_pages.
397 *
398 * For now it is simpler to just free the pages.
399 */
400 kimage_free_page_list(&extra_pages);
401
402 return pages;
403}
404
405static struct page *kimage_alloc_crash_control_pages(struct kimage *image,
406 unsigned int order)
407{
408 /* Control pages are special, they are the intermediaries
409 * that are needed while we copy the rest of the pages
410 * to their final resting place. As such they must
411 * not conflict with either the destination addresses
412 * or memory the kernel is already using.
413 *
414 * Control pages are also the only pags we must allocate
415 * when loading a crash kernel. All of the other pages
416 * are specified by the segments and we just memcpy
417 * into them directly.
418 *
419 * The only case where we really need more than one of
420 * these are for architectures where we cannot disable
421 * the MMU and must instead generate an identity mapped
422 * page table for all of the memory.
423 *
424 * Given the low demand this implements a very simple
425 * allocator that finds the first hole of the appropriate
426 * size in the reserved memory region, and allocates all
427 * of the memory up to and including the hole.
428 */
429 unsigned long hole_start, hole_end, size;
430 struct page *pages;
431
432 pages = NULL;
433 size = (1 << order) << PAGE_SHIFT;
434 hole_start = (image->control_page + (size - 1)) & ~(size - 1);
435 hole_end = hole_start + size - 1;
436 while (hole_end <= crashk_res.end) {
437 unsigned long i;
438
zhong jiang8e53c072016-12-14 15:04:23 -0800439 cond_resched();
440
Dave Young2965faa2015-09-09 15:38:55 -0700441 if (hole_end > KEXEC_CRASH_CONTROL_MEMORY_LIMIT)
442 break;
443 /* See if I overlap any of the segments */
444 for (i = 0; i < image->nr_segments; i++) {
445 unsigned long mstart, mend;
446
447 mstart = image->segment[i].mem;
448 mend = mstart + image->segment[i].memsz - 1;
449 if ((hole_end >= mstart) && (hole_start <= mend)) {
450 /* Advance the hole to the end of the segment */
451 hole_start = (mend + (size - 1)) & ~(size - 1);
452 hole_end = hole_start + size - 1;
453 break;
454 }
455 }
456 /* If I don't overlap any segments I have found my hole! */
457 if (i == image->nr_segments) {
458 pages = pfn_to_page(hole_start >> PAGE_SHIFT);
Minfei Huang04e99492015-09-09 15:38:58 -0700459 image->control_page = hole_end;
Dave Young2965faa2015-09-09 15:38:55 -0700460 break;
461 }
462 }
Dave Young2965faa2015-09-09 15:38:55 -0700463
464 return pages;
465}
466
467
468struct page *kimage_alloc_control_pages(struct kimage *image,
469 unsigned int order)
470{
471 struct page *pages = NULL;
472
473 switch (image->type) {
474 case KEXEC_TYPE_DEFAULT:
475 pages = kimage_alloc_normal_control_pages(image, order);
476 break;
477 case KEXEC_TYPE_CRASH:
478 pages = kimage_alloc_crash_control_pages(image, order);
479 break;
480 }
481
482 return pages;
483}
484
485static int kimage_add_entry(struct kimage *image, kimage_entry_t entry)
486{
487 if (*image->entry != 0)
488 image->entry++;
489
490 if (image->entry == image->last_entry) {
491 kimage_entry_t *ind_page;
492 struct page *page;
493
494 page = kimage_alloc_page(image, GFP_KERNEL, KIMAGE_NO_DEST);
495 if (!page)
496 return -ENOMEM;
497
498 ind_page = page_address(page);
Russell King43546d82016-08-02 14:06:04 -0700499 *image->entry = virt_to_boot_phys(ind_page) | IND_INDIRECTION;
Dave Young2965faa2015-09-09 15:38:55 -0700500 image->entry = ind_page;
501 image->last_entry = ind_page +
502 ((PAGE_SIZE/sizeof(kimage_entry_t)) - 1);
503 }
504 *image->entry = entry;
505 image->entry++;
506 *image->entry = 0;
507
508 return 0;
509}
510
511static int kimage_set_destination(struct kimage *image,
512 unsigned long destination)
513{
514 int result;
515
516 destination &= PAGE_MASK;
517 result = kimage_add_entry(image, destination | IND_DESTINATION);
518
519 return result;
520}
521
522
523static int kimage_add_page(struct kimage *image, unsigned long page)
524{
525 int result;
526
527 page &= PAGE_MASK;
528 result = kimage_add_entry(image, page | IND_SOURCE);
529
530 return result;
531}
532
533
534static void kimage_free_extra_pages(struct kimage *image)
535{
536 /* Walk through and free any extra destination pages I may have */
537 kimage_free_page_list(&image->dest_pages);
538
539 /* Walk through and free any unusable pages I have cached */
540 kimage_free_page_list(&image->unusable_pages);
541
542}
543void kimage_terminate(struct kimage *image)
544{
545 if (*image->entry != 0)
546 image->entry++;
547
548 *image->entry = IND_DONE;
549}
550
551#define for_each_kimage_entry(image, ptr, entry) \
552 for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE); \
553 ptr = (entry & IND_INDIRECTION) ? \
Russell King43546d82016-08-02 14:06:04 -0700554 boot_phys_to_virt((entry & PAGE_MASK)) : ptr + 1)
Dave Young2965faa2015-09-09 15:38:55 -0700555
556static void kimage_free_entry(kimage_entry_t entry)
557{
558 struct page *page;
559
Russell King43546d82016-08-02 14:06:04 -0700560 page = boot_pfn_to_page(entry >> PAGE_SHIFT);
Dave Young2965faa2015-09-09 15:38:55 -0700561 kimage_free_pages(page);
562}
563
564void kimage_free(struct kimage *image)
565{
566 kimage_entry_t *ptr, entry;
567 kimage_entry_t ind = 0;
568
569 if (!image)
570 return;
571
572 kimage_free_extra_pages(image);
573 for_each_kimage_entry(image, ptr, entry) {
574 if (entry & IND_INDIRECTION) {
575 /* Free the previous indirection page */
576 if (ind & IND_INDIRECTION)
577 kimage_free_entry(ind);
578 /* Save this indirection page until we are
579 * done with it.
580 */
581 ind = entry;
582 } else if (entry & IND_SOURCE)
583 kimage_free_entry(entry);
584 }
585 /* Free the final indirection page */
586 if (ind & IND_INDIRECTION)
587 kimage_free_entry(ind);
588
589 /* Handle any machine specific cleanup */
590 machine_kexec_cleanup(image);
591
592 /* Free the kexec control pages... */
593 kimage_free_page_list(&image->control_pages);
594
595 /*
596 * Free up any temporary buffers allocated. This might hit if
597 * error occurred much later after buffer allocation.
598 */
599 if (image->file_mode)
600 kimage_file_post_load_cleanup(image);
601
602 kfree(image);
603}
604
605static kimage_entry_t *kimage_dst_used(struct kimage *image,
606 unsigned long page)
607{
608 kimage_entry_t *ptr, entry;
609 unsigned long destination = 0;
610
611 for_each_kimage_entry(image, ptr, entry) {
612 if (entry & IND_DESTINATION)
613 destination = entry & PAGE_MASK;
614 else if (entry & IND_SOURCE) {
615 if (page == destination)
616 return ptr;
617 destination += PAGE_SIZE;
618 }
619 }
620
621 return NULL;
622}
623
624static struct page *kimage_alloc_page(struct kimage *image,
625 gfp_t gfp_mask,
626 unsigned long destination)
627{
628 /*
629 * Here we implement safeguards to ensure that a source page
630 * is not copied to its destination page before the data on
631 * the destination page is no longer useful.
632 *
633 * To do this we maintain the invariant that a source page is
634 * either its own destination page, or it is not a
635 * destination page at all.
636 *
637 * That is slightly stronger than required, but the proof
638 * that no problems will not occur is trivial, and the
639 * implementation is simply to verify.
640 *
641 * When allocating all pages normally this algorithm will run
642 * in O(N) time, but in the worst case it will run in O(N^2)
643 * time. If the runtime is a problem the data structures can
644 * be fixed.
645 */
646 struct page *page;
647 unsigned long addr;
648
649 /*
650 * Walk through the list of destination pages, and see if I
651 * have a match.
652 */
653 list_for_each_entry(page, &image->dest_pages, lru) {
Russell King43546d82016-08-02 14:06:04 -0700654 addr = page_to_boot_pfn(page) << PAGE_SHIFT;
Dave Young2965faa2015-09-09 15:38:55 -0700655 if (addr == destination) {
656 list_del(&page->lru);
657 return page;
658 }
659 }
660 page = NULL;
661 while (1) {
662 kimage_entry_t *old;
663
664 /* Allocate a page, if we run out of memory give up */
665 page = kimage_alloc_pages(gfp_mask, 0);
666 if (!page)
667 return NULL;
668 /* If the page cannot be used file it away */
Russell King43546d82016-08-02 14:06:04 -0700669 if (page_to_boot_pfn(page) >
Dave Young2965faa2015-09-09 15:38:55 -0700670 (KEXEC_SOURCE_MEMORY_LIMIT >> PAGE_SHIFT)) {
671 list_add(&page->lru, &image->unusable_pages);
672 continue;
673 }
Russell King43546d82016-08-02 14:06:04 -0700674 addr = page_to_boot_pfn(page) << PAGE_SHIFT;
Dave Young2965faa2015-09-09 15:38:55 -0700675
676 /* If it is the destination page we want use it */
677 if (addr == destination)
678 break;
679
680 /* If the page is not a destination page use it */
681 if (!kimage_is_destination_range(image, addr,
682 addr + PAGE_SIZE))
683 break;
684
685 /*
686 * I know that the page is someones destination page.
687 * See if there is already a source page for this
688 * destination page. And if so swap the source pages.
689 */
690 old = kimage_dst_used(image, addr);
691 if (old) {
692 /* If so move it */
693 unsigned long old_addr;
694 struct page *old_page;
695
696 old_addr = *old & PAGE_MASK;
Russell King43546d82016-08-02 14:06:04 -0700697 old_page = boot_pfn_to_page(old_addr >> PAGE_SHIFT);
Dave Young2965faa2015-09-09 15:38:55 -0700698 copy_highpage(page, old_page);
699 *old = addr | (*old & ~PAGE_MASK);
700
701 /* The old page I have found cannot be a
702 * destination page, so return it if it's
703 * gfp_flags honor the ones passed in.
704 */
705 if (!(gfp_mask & __GFP_HIGHMEM) &&
706 PageHighMem(old_page)) {
707 kimage_free_pages(old_page);
708 continue;
709 }
710 addr = old_addr;
711 page = old_page;
712 break;
713 }
714 /* Place the page on the destination list, to be used later */
715 list_add(&page->lru, &image->dest_pages);
716 }
717
718 return page;
719}
720
721static int kimage_load_normal_segment(struct kimage *image,
722 struct kexec_segment *segment)
723{
724 unsigned long maddr;
725 size_t ubytes, mbytes;
726 int result;
727 unsigned char __user *buf = NULL;
728 unsigned char *kbuf = NULL;
729
730 result = 0;
731 if (image->file_mode)
732 kbuf = segment->kbuf;
733 else
734 buf = segment->buf;
735 ubytes = segment->bufsz;
736 mbytes = segment->memsz;
737 maddr = segment->mem;
738
739 result = kimage_set_destination(image, maddr);
740 if (result < 0)
741 goto out;
742
743 while (mbytes) {
744 struct page *page;
745 char *ptr;
746 size_t uchunk, mchunk;
747
748 page = kimage_alloc_page(image, GFP_HIGHUSER, maddr);
749 if (!page) {
750 result = -ENOMEM;
751 goto out;
752 }
Russell King43546d82016-08-02 14:06:04 -0700753 result = kimage_add_page(image, page_to_boot_pfn(page)
Dave Young2965faa2015-09-09 15:38:55 -0700754 << PAGE_SHIFT);
755 if (result < 0)
756 goto out;
757
758 ptr = kmap(page);
759 /* Start with a clear page */
760 clear_page(ptr);
761 ptr += maddr & ~PAGE_MASK;
762 mchunk = min_t(size_t, mbytes,
763 PAGE_SIZE - (maddr & ~PAGE_MASK));
764 uchunk = min(ubytes, mchunk);
765
766 /* For file based kexec, source pages are in kernel memory */
767 if (image->file_mode)
768 memcpy(ptr, kbuf, uchunk);
769 else
770 result = copy_from_user(ptr, buf, uchunk);
771 kunmap(page);
772 if (result) {
773 result = -EFAULT;
774 goto out;
775 }
776 ubytes -= uchunk;
777 maddr += mchunk;
778 if (image->file_mode)
779 kbuf += mchunk;
780 else
781 buf += mchunk;
782 mbytes -= mchunk;
783 }
784out:
785 return result;
786}
787
788static int kimage_load_crash_segment(struct kimage *image,
789 struct kexec_segment *segment)
790{
791 /* For crash dumps kernels we simply copy the data from
792 * user space to it's destination.
793 * We do things a page at a time for the sake of kmap.
794 */
795 unsigned long maddr;
796 size_t ubytes, mbytes;
797 int result;
798 unsigned char __user *buf = NULL;
799 unsigned char *kbuf = NULL;
800
801 result = 0;
802 if (image->file_mode)
803 kbuf = segment->kbuf;
804 else
805 buf = segment->buf;
806 ubytes = segment->bufsz;
807 mbytes = segment->memsz;
808 maddr = segment->mem;
809 while (mbytes) {
810 struct page *page;
811 char *ptr;
812 size_t uchunk, mchunk;
813
Russell King43546d82016-08-02 14:06:04 -0700814 page = boot_pfn_to_page(maddr >> PAGE_SHIFT);
Dave Young2965faa2015-09-09 15:38:55 -0700815 if (!page) {
816 result = -ENOMEM;
817 goto out;
818 }
819 ptr = kmap(page);
820 ptr += maddr & ~PAGE_MASK;
821 mchunk = min_t(size_t, mbytes,
822 PAGE_SIZE - (maddr & ~PAGE_MASK));
823 uchunk = min(ubytes, mchunk);
824 if (mchunk > uchunk) {
825 /* Zero the trailing part of the page */
826 memset(ptr + uchunk, 0, mchunk - uchunk);
827 }
828
829 /* For file based kexec, source pages are in kernel memory */
830 if (image->file_mode)
831 memcpy(ptr, kbuf, uchunk);
832 else
833 result = copy_from_user(ptr, buf, uchunk);
834 kexec_flush_icache_page(page);
835 kunmap(page);
836 if (result) {
837 result = -EFAULT;
838 goto out;
839 }
840 ubytes -= uchunk;
841 maddr += mchunk;
842 if (image->file_mode)
843 kbuf += mchunk;
844 else
845 buf += mchunk;
846 mbytes -= mchunk;
847 }
848out:
849 return result;
850}
851
852int kimage_load_segment(struct kimage *image,
853 struct kexec_segment *segment)
854{
855 int result = -ENOMEM;
856
857 switch (image->type) {
858 case KEXEC_TYPE_DEFAULT:
859 result = kimage_load_normal_segment(image, segment);
860 break;
861 case KEXEC_TYPE_CRASH:
862 result = kimage_load_crash_segment(image, segment);
863 break;
864 }
865
866 return result;
867}
868
869struct kimage *kexec_image;
870struct kimage *kexec_crash_image;
871int kexec_load_disabled;
872
Hidehiro Kawai7bbee5c2015-12-14 11:19:11 +0100873/*
874 * No panic_cpu check version of crash_kexec(). This function is called
875 * only when panic_cpu holds the current CPU number; this is the only CPU
876 * which processes crash_kexec routines.
877 */
Josh Poimboeufc207aee2017-06-28 10:11:06 -0500878void __noclone __crash_kexec(struct pt_regs *regs)
Dave Young2965faa2015-09-09 15:38:55 -0700879{
880 /* Take the kexec_mutex here to prevent sys_kexec_load
881 * running on one cpu from replacing the crash kernel
882 * we are using after a panic on a different cpu.
883 *
884 * If the crash kernel was not located in a fixed area
885 * of memory the xchg(&kexec_crash_image) would be
886 * sufficient. But since I reuse the memory...
887 */
888 if (mutex_trylock(&kexec_mutex)) {
889 if (kexec_crash_image) {
890 struct pt_regs fixed_regs;
891
892 crash_setup_regs(&fixed_regs, regs);
893 crash_save_vmcoreinfo();
894 machine_crash_shutdown(&fixed_regs);
895 machine_kexec(kexec_crash_image);
896 }
897 mutex_unlock(&kexec_mutex);
898 }
899}
Josh Poimboeufc207aee2017-06-28 10:11:06 -0500900STACK_FRAME_NON_STANDARD(__crash_kexec);
Dave Young2965faa2015-09-09 15:38:55 -0700901
Hidehiro Kawai7bbee5c2015-12-14 11:19:11 +0100902void crash_kexec(struct pt_regs *regs)
903{
904 int old_cpu, this_cpu;
905
906 /*
907 * Only one CPU is allowed to execute the crash_kexec() code as with
908 * panic(). Otherwise parallel calls of panic() and crash_kexec()
909 * may stop each other. To exclude them, we use panic_cpu here too.
910 */
911 this_cpu = raw_smp_processor_id();
912 old_cpu = atomic_cmpxchg(&panic_cpu, PANIC_CPU_INVALID, this_cpu);
913 if (old_cpu == PANIC_CPU_INVALID) {
914 /* This is the 1st CPU which comes here, so go ahead. */
Sergey Senozhatskyf92bac32016-12-27 23:16:05 +0900915 printk_safe_flush_on_panic();
Hidehiro Kawai7bbee5c2015-12-14 11:19:11 +0100916 __crash_kexec(regs);
917
918 /*
919 * Reset panic_cpu to allow another panic()/crash_kexec()
920 * call.
921 */
922 atomic_set(&panic_cpu, PANIC_CPU_INVALID);
923 }
924}
925
Dave Young2965faa2015-09-09 15:38:55 -0700926size_t crash_get_memory_size(void)
927{
928 size_t size = 0;
929
930 mutex_lock(&kexec_mutex);
931 if (crashk_res.end != crashk_res.start)
932 size = resource_size(&crashk_res);
933 mutex_unlock(&kexec_mutex);
934 return size;
935}
936
937void __weak crash_free_reserved_phys_range(unsigned long begin,
938 unsigned long end)
939{
940 unsigned long addr;
941
942 for (addr = begin; addr < end; addr += PAGE_SIZE)
Russell King43546d82016-08-02 14:06:04 -0700943 free_reserved_page(boot_pfn_to_page(addr >> PAGE_SHIFT));
Dave Young2965faa2015-09-09 15:38:55 -0700944}
945
946int crash_shrink_memory(unsigned long new_size)
947{
948 int ret = 0;
949 unsigned long start, end;
950 unsigned long old_size;
951 struct resource *ram_res;
952
953 mutex_lock(&kexec_mutex);
954
955 if (kexec_crash_image) {
956 ret = -ENOENT;
957 goto unlock;
958 }
959 start = crashk_res.start;
960 end = crashk_res.end;
961 old_size = (end == 0) ? 0 : end - start + 1;
962 if (new_size >= old_size) {
963 ret = (new_size == old_size) ? 0 : -EINVAL;
964 goto unlock;
965 }
966
967 ram_res = kzalloc(sizeof(*ram_res), GFP_KERNEL);
968 if (!ram_res) {
969 ret = -ENOMEM;
970 goto unlock;
971 }
972
973 start = roundup(start, KEXEC_CRASH_MEM_ALIGN);
974 end = roundup(start + new_size, KEXEC_CRASH_MEM_ALIGN);
975
Dave Young2965faa2015-09-09 15:38:55 -0700976 crash_free_reserved_phys_range(end, crashk_res.end);
977
978 if ((start == end) && (crashk_res.parent != NULL))
979 release_resource(&crashk_res);
980
981 ram_res->start = end;
982 ram_res->end = crashk_res.end;
Toshi Kani1a085d02016-01-26 21:57:23 +0100983 ram_res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM;
Dave Young2965faa2015-09-09 15:38:55 -0700984 ram_res->name = "System RAM";
985
986 crashk_res.end = end - 1;
987
988 insert_resource(&iomem_resource, ram_res);
Dave Young2965faa2015-09-09 15:38:55 -0700989
990unlock:
991 mutex_unlock(&kexec_mutex);
992 return ret;
993}
994
Dave Young2965faa2015-09-09 15:38:55 -0700995void crash_save_cpu(struct pt_regs *regs, int cpu)
996{
997 struct elf_prstatus prstatus;
998 u32 *buf;
999
1000 if ((cpu < 0) || (cpu >= nr_cpu_ids))
1001 return;
1002
1003 /* Using ELF notes here is opportunistic.
1004 * I need a well defined structure format
1005 * for the data I pass, and I need tags
1006 * on the data to indicate what information I have
1007 * squirrelled away. ELF notes happen to provide
1008 * all of that, so there is no need to invent something new.
1009 */
1010 buf = (u32 *)per_cpu_ptr(crash_notes, cpu);
1011 if (!buf)
1012 return;
1013 memset(&prstatus, 0, sizeof(prstatus));
1014 prstatus.pr_pid = current->pid;
1015 elf_core_copy_kernel_regs(&prstatus.pr_reg, regs);
1016 buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS,
1017 &prstatus, sizeof(prstatus));
1018 final_note(buf);
1019}
1020
1021static int __init crash_notes_memory_init(void)
1022{
1023 /* Allocate memory for saving cpu registers. */
Baoquan Hebbb78b82015-09-09 15:39:00 -07001024 size_t size, align;
1025
1026 /*
1027 * crash_notes could be allocated across 2 vmalloc pages when percpu
1028 * is vmalloc based . vmalloc doesn't guarantee 2 continuous vmalloc
1029 * pages are also on 2 continuous physical pages. In this case the
1030 * 2nd part of crash_notes in 2nd page could be lost since only the
1031 * starting address and size of crash_notes are exported through sysfs.
1032 * Here round up the size of crash_notes to the nearest power of two
1033 * and pass it to __alloc_percpu as align value. This can make sure
1034 * crash_notes is allocated inside one physical page.
1035 */
1036 size = sizeof(note_buf_t);
1037 align = min(roundup_pow_of_two(sizeof(note_buf_t)), PAGE_SIZE);
1038
1039 /*
1040 * Break compile if size is bigger than PAGE_SIZE since crash_notes
1041 * definitely will be in 2 pages with that.
1042 */
1043 BUILD_BUG_ON(size > PAGE_SIZE);
1044
1045 crash_notes = __alloc_percpu(size, align);
Dave Young2965faa2015-09-09 15:38:55 -07001046 if (!crash_notes) {
Minfei Huangde90a6b2015-11-06 16:32:45 -08001047 pr_warn("Memory allocation for saving cpu register states failed\n");
Dave Young2965faa2015-09-09 15:38:55 -07001048 return -ENOMEM;
1049 }
1050 return 0;
1051}
1052subsys_initcall(crash_notes_memory_init);
1053
1054
1055/*
Dave Young2965faa2015-09-09 15:38:55 -07001056 * Move into place and start executing a preloaded standalone
1057 * executable. If nothing was preloaded return an error.
1058 */
1059int kernel_kexec(void)
1060{
1061 int error = 0;
1062
1063 if (!mutex_trylock(&kexec_mutex))
1064 return -EBUSY;
1065 if (!kexec_image) {
1066 error = -EINVAL;
1067 goto Unlock;
1068 }
1069
1070#ifdef CONFIG_KEXEC_JUMP
1071 if (kexec_image->preserve_context) {
1072 lock_system_sleep();
1073 pm_prepare_console();
1074 error = freeze_processes();
1075 if (error) {
1076 error = -EBUSY;
1077 goto Restore_console;
1078 }
1079 suspend_console();
1080 error = dpm_suspend_start(PMSG_FREEZE);
1081 if (error)
1082 goto Resume_console;
1083 /* At this point, dpm_suspend_start() has been called,
1084 * but *not* dpm_suspend_end(). We *must* call
1085 * dpm_suspend_end() now. Otherwise, drivers for
1086 * some devices (e.g. interrupt controllers) become
1087 * desynchronized with the actual state of the
1088 * hardware at resume time, and evil weirdness ensues.
1089 */
1090 error = dpm_suspend_end(PMSG_FREEZE);
1091 if (error)
1092 goto Resume_devices;
1093 error = disable_nonboot_cpus();
1094 if (error)
1095 goto Enable_cpus;
1096 local_irq_disable();
1097 error = syscore_suspend();
1098 if (error)
1099 goto Enable_irqs;
1100 } else
1101#endif
1102 {
1103 kexec_in_progress = true;
1104 kernel_restart_prepare(NULL);
1105 migrate_to_reboot_cpu();
1106
1107 /*
1108 * migrate_to_reboot_cpu() disables CPU hotplug assuming that
1109 * no further code needs to use CPU hotplug (which is true in
1110 * the reboot case). However, the kexec path depends on using
1111 * CPU hotplug again; so re-enable it here.
1112 */
1113 cpu_hotplug_enable();
1114 pr_emerg("Starting new kernel\n");
1115 machine_shutdown();
1116 }
1117
1118 machine_kexec(kexec_image);
1119
1120#ifdef CONFIG_KEXEC_JUMP
1121 if (kexec_image->preserve_context) {
1122 syscore_resume();
1123 Enable_irqs:
1124 local_irq_enable();
1125 Enable_cpus:
1126 enable_nonboot_cpus();
1127 dpm_resume_start(PMSG_RESTORE);
1128 Resume_devices:
1129 dpm_resume_end(PMSG_RESTORE);
1130 Resume_console:
1131 resume_console();
1132 thaw_processes();
1133 Restore_console:
1134 pm_restore_console();
1135 unlock_system_sleep();
1136 }
1137#endif
1138
1139 Unlock:
1140 mutex_unlock(&kexec_mutex);
1141 return error;
1142}
1143
1144/*
Xunlei Pang7a0058e2016-05-23 16:24:22 -07001145 * Protection mechanism for crashkernel reserved memory after
1146 * the kdump kernel is loaded.
Dave Young2965faa2015-09-09 15:38:55 -07001147 *
1148 * Provide an empty default implementation here -- architecture
1149 * code may override this
1150 */
Xunlei Pang9b492cf2016-05-23 16:24:10 -07001151void __weak arch_kexec_protect_crashkres(void)
1152{}
1153
1154void __weak arch_kexec_unprotect_crashkres(void)
1155{}