Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 1 | /* |
| 2 | * early_res, could be used to replace bootmem |
| 3 | */ |
| 4 | #include <linux/kernel.h> |
| 5 | #include <linux/types.h> |
| 6 | #include <linux/init.h> |
| 7 | #include <linux/bootmem.h> |
| 8 | #include <linux/mm.h> |
| 9 | |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 10 | #include <asm/early_res.h> |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 11 | |
| 12 | /* |
| 13 | * Early reserved memory areas. |
| 14 | */ |
| 15 | /* |
| 16 | * need to make sure this one is bigger enough before |
Yinghai Lu | dd645ce | 2010-02-10 01:20:30 -0800 | [diff] [blame^] | 17 | * find_fw_memmap_area could be used |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 18 | */ |
| 19 | #define MAX_EARLY_RES_X 32 |
| 20 | |
| 21 | struct early_res { |
| 22 | u64 start, end; |
| 23 | char name[15]; |
| 24 | char overlap_ok; |
| 25 | }; |
| 26 | static struct early_res early_res_x[MAX_EARLY_RES_X] __initdata; |
| 27 | |
| 28 | static int max_early_res __initdata = MAX_EARLY_RES_X; |
| 29 | static struct early_res *early_res __initdata = &early_res_x[0]; |
| 30 | static int early_res_count __initdata; |
| 31 | |
| 32 | static int __init find_overlapped_early(u64 start, u64 end) |
| 33 | { |
| 34 | int i; |
| 35 | struct early_res *r; |
| 36 | |
| 37 | for (i = 0; i < max_early_res && early_res[i].end; i++) { |
| 38 | r = &early_res[i]; |
| 39 | if (end > r->start && start < r->end) |
| 40 | break; |
| 41 | } |
| 42 | |
| 43 | return i; |
| 44 | } |
| 45 | |
| 46 | /* |
| 47 | * Drop the i-th range from the early reservation map, |
| 48 | * by copying any higher ranges down one over it, and |
| 49 | * clearing what had been the last slot. |
| 50 | */ |
| 51 | static void __init drop_range(int i) |
| 52 | { |
| 53 | int j; |
| 54 | |
| 55 | for (j = i + 1; j < max_early_res && early_res[j].end; j++) |
| 56 | ; |
| 57 | |
| 58 | memmove(&early_res[i], &early_res[i + 1], |
| 59 | (j - 1 - i) * sizeof(struct early_res)); |
| 60 | |
| 61 | early_res[j - 1].end = 0; |
| 62 | early_res_count--; |
| 63 | } |
| 64 | |
| 65 | /* |
| 66 | * Split any existing ranges that: |
| 67 | * 1) are marked 'overlap_ok', and |
| 68 | * 2) overlap with the stated range [start, end) |
| 69 | * into whatever portion (if any) of the existing range is entirely |
| 70 | * below or entirely above the stated range. Drop the portion |
| 71 | * of the existing range that overlaps with the stated range, |
| 72 | * which will allow the caller of this routine to then add that |
| 73 | * stated range without conflicting with any existing range. |
| 74 | */ |
| 75 | static void __init drop_overlaps_that_are_ok(u64 start, u64 end) |
| 76 | { |
| 77 | int i; |
| 78 | struct early_res *r; |
| 79 | u64 lower_start, lower_end; |
| 80 | u64 upper_start, upper_end; |
| 81 | char name[15]; |
| 82 | |
| 83 | for (i = 0; i < max_early_res && early_res[i].end; i++) { |
| 84 | r = &early_res[i]; |
| 85 | |
| 86 | /* Continue past non-overlapping ranges */ |
| 87 | if (end <= r->start || start >= r->end) |
| 88 | continue; |
| 89 | |
| 90 | /* |
| 91 | * Leave non-ok overlaps as is; let caller |
| 92 | * panic "Overlapping early reservations" |
| 93 | * when it hits this overlap. |
| 94 | */ |
| 95 | if (!r->overlap_ok) |
| 96 | return; |
| 97 | |
| 98 | /* |
| 99 | * We have an ok overlap. We will drop it from the early |
| 100 | * reservation map, and add back in any non-overlapping |
| 101 | * portions (lower or upper) as separate, overlap_ok, |
| 102 | * non-overlapping ranges. |
| 103 | */ |
| 104 | |
| 105 | /* 1. Note any non-overlapping (lower or upper) ranges. */ |
| 106 | strncpy(name, r->name, sizeof(name) - 1); |
| 107 | |
| 108 | lower_start = lower_end = 0; |
| 109 | upper_start = upper_end = 0; |
| 110 | if (r->start < start) { |
| 111 | lower_start = r->start; |
| 112 | lower_end = start; |
| 113 | } |
| 114 | if (r->end > end) { |
| 115 | upper_start = end; |
| 116 | upper_end = r->end; |
| 117 | } |
| 118 | |
| 119 | /* 2. Drop the original ok overlapping range */ |
| 120 | drop_range(i); |
| 121 | |
| 122 | i--; /* resume for-loop on copied down entry */ |
| 123 | |
| 124 | /* 3. Add back in any non-overlapping ranges. */ |
| 125 | if (lower_end) |
| 126 | reserve_early_overlap_ok(lower_start, lower_end, name); |
| 127 | if (upper_end) |
| 128 | reserve_early_overlap_ok(upper_start, upper_end, name); |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | static void __init __reserve_early(u64 start, u64 end, char *name, |
| 133 | int overlap_ok) |
| 134 | { |
| 135 | int i; |
| 136 | struct early_res *r; |
| 137 | |
| 138 | i = find_overlapped_early(start, end); |
| 139 | if (i >= max_early_res) |
| 140 | panic("Too many early reservations"); |
| 141 | r = &early_res[i]; |
| 142 | if (r->end) |
| 143 | panic("Overlapping early reservations " |
| 144 | "%llx-%llx %s to %llx-%llx %s\n", |
| 145 | start, end - 1, name ? name : "", r->start, |
| 146 | r->end - 1, r->name); |
| 147 | r->start = start; |
| 148 | r->end = end; |
| 149 | r->overlap_ok = overlap_ok; |
| 150 | if (name) |
| 151 | strncpy(r->name, name, sizeof(r->name) - 1); |
| 152 | early_res_count++; |
| 153 | } |
| 154 | |
| 155 | /* |
| 156 | * A few early reservtations come here. |
| 157 | * |
| 158 | * The 'overlap_ok' in the name of this routine does -not- mean it |
| 159 | * is ok for these reservations to overlap an earlier reservation. |
| 160 | * Rather it means that it is ok for subsequent reservations to |
| 161 | * overlap this one. |
| 162 | * |
| 163 | * Use this entry point to reserve early ranges when you are doing |
| 164 | * so out of "Paranoia", reserving perhaps more memory than you need, |
| 165 | * just in case, and don't mind a subsequent overlapping reservation |
| 166 | * that is known to be needed. |
| 167 | * |
| 168 | * The drop_overlaps_that_are_ok() call here isn't really needed. |
| 169 | * It would be needed if we had two colliding 'overlap_ok' |
| 170 | * reservations, so that the second such would not panic on the |
| 171 | * overlap with the first. We don't have any such as of this |
| 172 | * writing, but might as well tolerate such if it happens in |
| 173 | * the future. |
| 174 | */ |
| 175 | void __init reserve_early_overlap_ok(u64 start, u64 end, char *name) |
| 176 | { |
| 177 | drop_overlaps_that_are_ok(start, end); |
| 178 | __reserve_early(start, end, name, 1); |
| 179 | } |
| 180 | |
Yinghai Lu | dd645ce | 2010-02-10 01:20:30 -0800 | [diff] [blame^] | 181 | u64 __init __weak find_fw_memmap_area(u64 start, u64 end, u64 size, u64 align) |
| 182 | { |
| 183 | panic("should have find_fw_memmap_area defined with arch"); |
| 184 | |
| 185 | return -1ULL; |
| 186 | } |
| 187 | |
Yinghai Lu | 53db62a | 2010-02-10 01:20:27 -0800 | [diff] [blame] | 188 | static void __init __check_and_double_early_res(u64 ex_start, u64 ex_end) |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 189 | { |
Yinghai Lu | 53db62a | 2010-02-10 01:20:27 -0800 | [diff] [blame] | 190 | u64 start, end, size, mem; |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 191 | struct early_res *new; |
| 192 | |
| 193 | /* do we have enough slots left ? */ |
| 194 | if ((max_early_res - early_res_count) > max(max_early_res/8, 2)) |
| 195 | return; |
| 196 | |
| 197 | /* double it */ |
Yinghai Lu | 53db62a | 2010-02-10 01:20:27 -0800 | [diff] [blame] | 198 | mem = -1ULL; |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 199 | size = sizeof(struct early_res) * max_early_res * 2; |
Yinghai Lu | 53db62a | 2010-02-10 01:20:27 -0800 | [diff] [blame] | 200 | if (early_res == early_res_x) |
| 201 | start = 0; |
| 202 | else |
| 203 | start = early_res[0].end; |
| 204 | end = ex_start; |
| 205 | if (start + size < end) |
Yinghai Lu | dd645ce | 2010-02-10 01:20:30 -0800 | [diff] [blame^] | 206 | mem = find_fw_memmap_area(start, end, size, |
Yinghai Lu | 53db62a | 2010-02-10 01:20:27 -0800 | [diff] [blame] | 207 | sizeof(struct early_res)); |
| 208 | if (mem == -1ULL) { |
| 209 | start = ex_end; |
| 210 | end = max_pfn_mapped << PAGE_SHIFT; |
| 211 | if (start + size < end) |
Yinghai Lu | dd645ce | 2010-02-10 01:20:30 -0800 | [diff] [blame^] | 212 | mem = find_fw_memmap_area(start, end, size, |
Yinghai Lu | 53db62a | 2010-02-10 01:20:27 -0800 | [diff] [blame] | 213 | sizeof(struct early_res)); |
| 214 | } |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 215 | if (mem == -1ULL) |
| 216 | panic("can not find more space for early_res array"); |
| 217 | |
| 218 | new = __va(mem); |
| 219 | /* save the first one for own */ |
| 220 | new[0].start = mem; |
| 221 | new[0].end = mem + size; |
| 222 | new[0].overlap_ok = 0; |
| 223 | /* copy old to new */ |
| 224 | if (early_res == early_res_x) { |
| 225 | memcpy(&new[1], &early_res[0], |
| 226 | sizeof(struct early_res) * max_early_res); |
| 227 | memset(&new[max_early_res+1], 0, |
| 228 | sizeof(struct early_res) * (max_early_res - 1)); |
| 229 | early_res_count++; |
| 230 | } else { |
| 231 | memcpy(&new[1], &early_res[1], |
| 232 | sizeof(struct early_res) * (max_early_res - 1)); |
| 233 | memset(&new[max_early_res], 0, |
| 234 | sizeof(struct early_res) * max_early_res); |
| 235 | } |
| 236 | memset(&early_res[0], 0, sizeof(struct early_res) * max_early_res); |
| 237 | early_res = new; |
| 238 | max_early_res *= 2; |
| 239 | printk(KERN_DEBUG "early_res array is doubled to %d at [%llx - %llx]\n", |
| 240 | max_early_res, mem, mem + size - 1); |
| 241 | } |
| 242 | |
| 243 | /* |
| 244 | * Most early reservations come here. |
| 245 | * |
| 246 | * We first have drop_overlaps_that_are_ok() drop any pre-existing |
| 247 | * 'overlap_ok' ranges, so that we can then reserve this memory |
| 248 | * range without risk of panic'ing on an overlapping overlap_ok |
| 249 | * early reservation. |
| 250 | */ |
| 251 | void __init reserve_early(u64 start, u64 end, char *name) |
| 252 | { |
| 253 | if (start >= end) |
| 254 | return; |
| 255 | |
Yinghai Lu | 53db62a | 2010-02-10 01:20:27 -0800 | [diff] [blame] | 256 | __check_and_double_early_res(start, end); |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 257 | |
| 258 | drop_overlaps_that_are_ok(start, end); |
| 259 | __reserve_early(start, end, name, 0); |
| 260 | } |
| 261 | |
| 262 | void __init reserve_early_without_check(u64 start, u64 end, char *name) |
| 263 | { |
| 264 | struct early_res *r; |
| 265 | |
| 266 | if (start >= end) |
| 267 | return; |
| 268 | |
Yinghai Lu | 53db62a | 2010-02-10 01:20:27 -0800 | [diff] [blame] | 269 | __check_and_double_early_res(start, end); |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 270 | |
| 271 | r = &early_res[early_res_count]; |
| 272 | |
| 273 | r->start = start; |
| 274 | r->end = end; |
| 275 | r->overlap_ok = 0; |
| 276 | if (name) |
| 277 | strncpy(r->name, name, sizeof(r->name) - 1); |
| 278 | early_res_count++; |
| 279 | } |
| 280 | |
| 281 | void __init free_early(u64 start, u64 end) |
| 282 | { |
| 283 | struct early_res *r; |
| 284 | int i; |
| 285 | |
| 286 | i = find_overlapped_early(start, end); |
| 287 | r = &early_res[i]; |
| 288 | if (i >= max_early_res || r->end != end || r->start != start) |
| 289 | panic("free_early on not reserved area: %llx-%llx!", |
| 290 | start, end - 1); |
| 291 | |
| 292 | drop_range(i); |
| 293 | } |
| 294 | |
| 295 | #ifdef CONFIG_NO_BOOTMEM |
| 296 | static void __init subtract_early_res(struct range *range, int az) |
| 297 | { |
| 298 | int i, count; |
| 299 | u64 final_start, final_end; |
| 300 | int idx = 0; |
| 301 | |
| 302 | count = 0; |
| 303 | for (i = 0; i < max_early_res && early_res[i].end; i++) |
| 304 | count++; |
| 305 | |
| 306 | /* need to skip first one ?*/ |
| 307 | if (early_res != early_res_x) |
| 308 | idx = 1; |
| 309 | |
| 310 | #define DEBUG_PRINT_EARLY_RES 1 |
| 311 | |
| 312 | #if DEBUG_PRINT_EARLY_RES |
| 313 | printk(KERN_INFO "Subtract (%d early reservations)\n", count); |
| 314 | #endif |
| 315 | for (i = idx; i < count; i++) { |
| 316 | struct early_res *r = &early_res[i]; |
| 317 | #if DEBUG_PRINT_EARLY_RES |
| 318 | printk(KERN_INFO " #%d [%010llx - %010llx] %15s\n", i, |
| 319 | r->start, r->end, r->name); |
| 320 | #endif |
| 321 | final_start = PFN_DOWN(r->start); |
| 322 | final_end = PFN_UP(r->end); |
| 323 | if (final_start >= final_end) |
| 324 | continue; |
| 325 | subtract_range(range, az, final_start, final_end); |
| 326 | } |
| 327 | |
| 328 | } |
| 329 | |
| 330 | int __init get_free_all_memory_range(struct range **rangep, int nodeid) |
| 331 | { |
| 332 | int i, count; |
| 333 | u64 start = 0, end; |
| 334 | u64 size; |
| 335 | u64 mem; |
| 336 | struct range *range; |
| 337 | int nr_range; |
| 338 | |
| 339 | count = 0; |
| 340 | for (i = 0; i < max_early_res && early_res[i].end; i++) |
| 341 | count++; |
| 342 | |
| 343 | count *= 2; |
| 344 | |
| 345 | size = sizeof(struct range) * count; |
| 346 | #ifdef MAX_DMA32_PFN |
| 347 | if (max_pfn_mapped > MAX_DMA32_PFN) |
| 348 | start = MAX_DMA32_PFN << PAGE_SHIFT; |
| 349 | #endif |
| 350 | end = max_pfn_mapped << PAGE_SHIFT; |
Yinghai Lu | dd645ce | 2010-02-10 01:20:30 -0800 | [diff] [blame^] | 351 | mem = find_fw_memmap_area(start, end, size, sizeof(struct range)); |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 352 | if (mem == -1ULL) |
| 353 | panic("can not find more space for range free"); |
| 354 | |
| 355 | range = __va(mem); |
| 356 | /* use early_node_map[] and early_res to get range array at first */ |
| 357 | memset(range, 0, size); |
| 358 | nr_range = 0; |
| 359 | |
| 360 | /* need to go over early_node_map to find out good range for node */ |
| 361 | nr_range = add_from_early_node_map(range, count, nr_range, nodeid); |
Yinghai Lu | 59be5a8 | 2010-02-10 01:20:28 -0800 | [diff] [blame] | 362 | #ifdef CONFIG_X86_32 |
| 363 | subtract_range(range, count, max_low_pfn, -1ULL); |
| 364 | #endif |
Yinghai Lu | a678c2b | 2010-02-10 01:20:24 -0800 | [diff] [blame] | 365 | subtract_early_res(range, count); |
| 366 | nr_range = clean_sort_range(range, count); |
| 367 | |
| 368 | /* need to clear it ? */ |
| 369 | if (nodeid == MAX_NUMNODES) { |
| 370 | memset(&early_res[0], 0, |
| 371 | sizeof(struct early_res) * max_early_res); |
| 372 | early_res = NULL; |
| 373 | max_early_res = 0; |
| 374 | } |
| 375 | |
| 376 | *rangep = range; |
| 377 | return nr_range; |
| 378 | } |
| 379 | #else |
| 380 | void __init early_res_to_bootmem(u64 start, u64 end) |
| 381 | { |
| 382 | int i, count; |
| 383 | u64 final_start, final_end; |
| 384 | int idx = 0; |
| 385 | |
| 386 | count = 0; |
| 387 | for (i = 0; i < max_early_res && early_res[i].end; i++) |
| 388 | count++; |
| 389 | |
| 390 | /* need to skip first one ?*/ |
| 391 | if (early_res != early_res_x) |
| 392 | idx = 1; |
| 393 | |
| 394 | printk(KERN_INFO "(%d/%d early reservations) ==> bootmem [%010llx - %010llx]\n", |
| 395 | count - idx, max_early_res, start, end); |
| 396 | for (i = idx; i < count; i++) { |
| 397 | struct early_res *r = &early_res[i]; |
| 398 | printk(KERN_INFO " #%d [%010llx - %010llx] %16s", i, |
| 399 | r->start, r->end, r->name); |
| 400 | final_start = max(start, r->start); |
| 401 | final_end = min(end, r->end); |
| 402 | if (final_start >= final_end) { |
| 403 | printk(KERN_CONT "\n"); |
| 404 | continue; |
| 405 | } |
| 406 | printk(KERN_CONT " ==> [%010llx - %010llx]\n", |
| 407 | final_start, final_end); |
| 408 | reserve_bootmem_generic(final_start, final_end - final_start, |
| 409 | BOOTMEM_DEFAULT); |
| 410 | } |
| 411 | /* clear them */ |
| 412 | memset(&early_res[0], 0, sizeof(struct early_res) * max_early_res); |
| 413 | early_res = NULL; |
| 414 | max_early_res = 0; |
| 415 | early_res_count = 0; |
| 416 | } |
| 417 | #endif |
| 418 | |
| 419 | /* Check for already reserved areas */ |
| 420 | static inline int __init bad_addr(u64 *addrp, u64 size, u64 align) |
| 421 | { |
| 422 | int i; |
| 423 | u64 addr = *addrp; |
| 424 | int changed = 0; |
| 425 | struct early_res *r; |
| 426 | again: |
| 427 | i = find_overlapped_early(addr, addr + size); |
| 428 | r = &early_res[i]; |
| 429 | if (i < max_early_res && r->end) { |
| 430 | *addrp = addr = round_up(r->end, align); |
| 431 | changed = 1; |
| 432 | goto again; |
| 433 | } |
| 434 | return changed; |
| 435 | } |
| 436 | |
| 437 | /* Check for already reserved areas */ |
| 438 | static inline int __init bad_addr_size(u64 *addrp, u64 *sizep, u64 align) |
| 439 | { |
| 440 | int i; |
| 441 | u64 addr = *addrp, last; |
| 442 | u64 size = *sizep; |
| 443 | int changed = 0; |
| 444 | again: |
| 445 | last = addr + size; |
| 446 | for (i = 0; i < max_early_res && early_res[i].end; i++) { |
| 447 | struct early_res *r = &early_res[i]; |
| 448 | if (last > r->start && addr < r->start) { |
| 449 | size = r->start - addr; |
| 450 | changed = 1; |
| 451 | goto again; |
| 452 | } |
| 453 | if (last > r->end && addr < r->end) { |
| 454 | addr = round_up(r->end, align); |
| 455 | size = last - addr; |
| 456 | changed = 1; |
| 457 | goto again; |
| 458 | } |
| 459 | if (last <= r->end && addr >= r->start) { |
| 460 | (*sizep)++; |
| 461 | return 0; |
| 462 | } |
| 463 | } |
| 464 | if (changed) { |
| 465 | *addrp = addr; |
| 466 | *sizep = size; |
| 467 | } |
| 468 | return changed; |
| 469 | } |
| 470 | |
| 471 | /* |
| 472 | * Find a free area with specified alignment in a specific range. |
| 473 | * only with the area.between start to end is active range from early_node_map |
| 474 | * so they are good as RAM |
| 475 | */ |
| 476 | u64 __init find_early_area(u64 ei_start, u64 ei_last, u64 start, u64 end, |
| 477 | u64 size, u64 align) |
| 478 | { |
| 479 | u64 addr, last; |
| 480 | |
| 481 | addr = round_up(ei_start, align); |
| 482 | if (addr < start) |
| 483 | addr = round_up(start, align); |
| 484 | if (addr >= ei_last) |
| 485 | goto out; |
| 486 | while (bad_addr(&addr, size, align) && addr+size <= ei_last) |
| 487 | ; |
| 488 | last = addr + size; |
| 489 | if (last > ei_last) |
| 490 | goto out; |
| 491 | if (last > end) |
| 492 | goto out; |
| 493 | |
| 494 | return addr; |
| 495 | |
| 496 | out: |
| 497 | return -1ULL; |
| 498 | } |
| 499 | |
Yinghai Lu | 7da657d | 2010-02-10 01:20:25 -0800 | [diff] [blame] | 500 | u64 __init find_early_area_size(u64 ei_start, u64 ei_last, u64 start, |
| 501 | u64 *sizep, u64 align) |
| 502 | { |
| 503 | u64 addr, last; |
| 504 | |
| 505 | addr = round_up(ei_start, align); |
| 506 | if (addr < start) |
| 507 | addr = round_up(start, align); |
| 508 | if (addr >= ei_last) |
| 509 | goto out; |
| 510 | *sizep = ei_last - addr; |
| 511 | while (bad_addr_size(&addr, sizep, align) && addr + *sizep <= ei_last) |
| 512 | ; |
| 513 | last = addr + *sizep; |
| 514 | if (last > ei_last) |
| 515 | goto out; |
| 516 | |
| 517 | return addr; |
| 518 | |
| 519 | out: |
| 520 | return -1ULL; |
| 521 | } |