Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * Firmware replacement code. |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 3 | * |
Pavel Machek | 8caac56 | 2008-11-26 17:15:27 +0100 | [diff] [blame] | 4 | * Work around broken BIOSes that don't set an aperture, only set the |
| 5 | * aperture in the AGP bridge, or set too small aperture. |
| 6 | * |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 7 | * If all fails map the aperture over some low memory. This is cheaper than |
| 8 | * doing bounce buffering. The memory is lost. This is done at early boot |
| 9 | * because only the bootmem allocator can allocate 32+MB. |
| 10 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | * Copyright 2002 Andi Kleen, SuSE Labs. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <linux/kernel.h> |
| 14 | #include <linux/types.h> |
| 15 | #include <linux/init.h> |
Yinghai Lu | 32e3f2b | 2010-12-17 16:58:40 -0800 | [diff] [blame] | 16 | #include <linux/memblock.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #include <linux/mmzone.h> |
| 18 | #include <linux/pci_ids.h> |
| 19 | #include <linux/pci.h> |
| 20 | #include <linux/bitops.h> |
Pavel Machek | 2050d45 | 2008-03-13 23:05:41 +0100 | [diff] [blame] | 21 | #include <linux/suspend.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 22 | #include <asm/e820.h> |
| 23 | #include <asm/io.h> |
FUJITA Tomonori | 46a7fa2 | 2008-07-11 10:23:42 +0900 | [diff] [blame] | 24 | #include <asm/iommu.h> |
Joerg Roedel | 395624f | 2007-10-24 12:49:47 +0200 | [diff] [blame] | 25 | #include <asm/gart.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | #include <asm/pci-direct.h> |
Andi Kleen | ca8642f | 2006-01-11 22:44:27 +0100 | [diff] [blame] | 27 | #include <asm/dma.h> |
Andreas Herrmann | 23ac4ae | 2010-09-17 18:03:43 +0200 | [diff] [blame] | 28 | #include <asm/amd_nb.h> |
FUJITA Tomonori | de95762 | 2009-11-10 19:46:14 +0900 | [diff] [blame] | 29 | #include <asm/x86_init.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | |
Joerg Roedel | c387aa3 | 2011-04-18 15:45:43 +0200 | [diff] [blame] | 31 | /* |
| 32 | * Using 512M as goal, in case kexec will load kernel_big |
| 33 | * that will do the on-position decompress, and could overlap with |
| 34 | * with the gart aperture that is used. |
| 35 | * Sequence: |
| 36 | * kernel_small |
| 37 | * ==> kexec (with kdump trigger path or gart still enabled) |
| 38 | * ==> kernel_small (gart area become e820_reserved) |
| 39 | * ==> kexec (with kdump trigger path or gart still enabled) |
| 40 | * ==> kerne_big (uncompressed size will be big than 64M or 128M) |
| 41 | * So don't use 512M below as gart iommu, leave the space for kernel |
| 42 | * code for safe. |
| 43 | */ |
| 44 | #define GART_MIN_ADDR (512ULL << 20) |
| 45 | #define GART_MAX_ADDR (1ULL << 32) |
| 46 | |
Joerg Roedel | 0440d4c | 2007-10-24 12:49:50 +0200 | [diff] [blame] | 47 | int gart_iommu_aperture; |
Pavel Machek | 7de6a4c | 2008-03-13 11:03:58 +0100 | [diff] [blame] | 48 | int gart_iommu_aperture_disabled __initdata; |
| 49 | int gart_iommu_aperture_allowed __initdata; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 50 | |
| 51 | int fallback_aper_order __initdata = 1; /* 64MB */ |
Pavel Machek | 7de6a4c | 2008-03-13 11:03:58 +0100 | [diff] [blame] | 52 | int fallback_aper_force __initdata; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 53 | |
| 54 | int fix_aperture __initdata = 1; |
| 55 | |
Andrew Morton | 42442ed | 2005-06-08 15:49:25 -0700 | [diff] [blame] | 56 | /* This code runs before the PCI subsystem is initialized, so just |
| 57 | access the northbridge directly. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 58 | |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 59 | static u32 __init allocate_aperture(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 60 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 61 | u32 aper_size; |
Yinghai Lu | 32e3f2b | 2010-12-17 16:58:40 -0800 | [diff] [blame] | 62 | unsigned long addr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | |
Yinghai Lu | 7677b2e | 2008-04-14 20:40:37 -0700 | [diff] [blame] | 64 | /* aper_size should <= 1G */ |
| 65 | if (fallback_aper_order > 5) |
| 66 | fallback_aper_order = 5; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 67 | aper_size = (32 * 1024 * 1024) << fallback_aper_order; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 68 | |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 69 | /* |
| 70 | * Aperture has to be naturally aligned. This means a 2GB aperture |
| 71 | * won't have much chance of finding a place in the lower 4GB of |
| 72 | * memory. Unfortunately we cannot move it up because that would |
| 73 | * make the IOMMU useless. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | */ |
Joerg Roedel | c387aa3 | 2011-04-18 15:45:43 +0200 | [diff] [blame] | 75 | addr = memblock_find_in_range(GART_MIN_ADDR, GART_MAX_ADDR, |
| 76 | aper_size, aper_size); |
Wang YanQing | 26bfc54 | 2013-04-16 09:37:34 +0800 | [diff] [blame] | 77 | if (!addr) { |
Yinghai Lu | 32e3f2b | 2010-12-17 16:58:40 -0800 | [diff] [blame] | 78 | printk(KERN_ERR |
| 79 | "Cannot allocate aperture memory hole (%lx,%uK)\n", |
| 80 | addr, aper_size>>10); |
| 81 | return 0; |
| 82 | } |
Tejun Heo | 24aa078 | 2011-07-12 11:16:06 +0200 | [diff] [blame] | 83 | memblock_reserve(addr, aper_size); |
Ingo Molnar | 31183ba | 2008-01-30 13:30:10 +0100 | [diff] [blame] | 84 | printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n", |
Yinghai Lu | 32e3f2b | 2010-12-17 16:58:40 -0800 | [diff] [blame] | 85 | aper_size >> 10, addr); |
Yinghai Lu | 32e3f2b | 2010-12-17 16:58:40 -0800 | [diff] [blame] | 86 | register_nosave_region(addr >> PAGE_SHIFT, |
| 87 | (addr+aper_size) >> PAGE_SHIFT); |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 88 | |
Yinghai Lu | 32e3f2b | 2010-12-17 16:58:40 -0800 | [diff] [blame] | 89 | return (u32)addr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 90 | } |
| 91 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 92 | |
Andrew Morton | 42442ed | 2005-06-08 15:49:25 -0700 | [diff] [blame] | 93 | /* Find a PCI capability */ |
Pavel Machek | dd564d0 | 2008-05-27 18:03:56 +0200 | [diff] [blame] | 94 | static u32 __init find_cap(int bus, int slot, int func, int cap) |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 95 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 96 | int bytes; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 97 | u8 pos; |
| 98 | |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 99 | if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 100 | PCI_STATUS_CAP_LIST)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 101 | return 0; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 102 | |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 103 | pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 104 | for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 105 | u8 id; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 106 | |
| 107 | pos &= ~3; |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 108 | id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 109 | if (id == 0xff) |
| 110 | break; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 111 | if (id == cap) |
| 112 | return pos; |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 113 | pos = read_pci_config_byte(bus, slot, func, |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 114 | pos+PCI_CAP_LIST_NEXT); |
| 115 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 116 | return 0; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 117 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 118 | |
| 119 | /* Read a standard AGPv3 bridge header */ |
Pavel Machek | dd564d0 | 2008-05-27 18:03:56 +0200 | [diff] [blame] | 120 | static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 121 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | u32 apsize; |
| 123 | u32 apsizereg; |
| 124 | int nbits; |
| 125 | u32 aper_low, aper_hi; |
| 126 | u64 aper; |
Yinghai Lu | 1edc1ab | 2008-04-13 01:11:41 -0700 | [diff] [blame] | 127 | u32 old_order; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 128 | |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 129 | printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func); |
| 130 | apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 131 | if (apsizereg == 0xffffffff) { |
Ingo Molnar | 31183ba | 2008-01-30 13:30:10 +0100 | [diff] [blame] | 132 | printk(KERN_ERR "APSIZE in AGP bridge unreadable\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 133 | return 0; |
| 134 | } |
| 135 | |
Yinghai Lu | 1edc1ab | 2008-04-13 01:11:41 -0700 | [diff] [blame] | 136 | /* old_order could be the value from NB gart setting */ |
| 137 | old_order = *order; |
| 138 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | apsize = apsizereg & 0xfff; |
| 140 | /* Some BIOS use weird encodings not in the AGPv3 table. */ |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 141 | if (apsize & 0xff) |
| 142 | apsize |= 0xf00; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 143 | nbits = hweight16(apsize); |
| 144 | *order = 7 - nbits; |
| 145 | if ((int)*order < 0) /* < 32MB */ |
| 146 | *order = 0; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 147 | |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 148 | aper_low = read_pci_config(bus, slot, func, 0x10); |
| 149 | aper_hi = read_pci_config(bus, slot, func, 0x14); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 150 | aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); |
| 151 | |
Yinghai Lu | 1edc1ab | 2008-04-13 01:11:41 -0700 | [diff] [blame] | 152 | /* |
| 153 | * On some sick chips, APSIZE is 0. It means it wants 4G |
| 154 | * so let double check that order, and lets trust AMD NB settings: |
| 155 | */ |
Yinghai Lu | 8c9fd91 | 2008-04-13 18:42:31 -0700 | [diff] [blame] | 156 | printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n", |
| 157 | aper, 32 << old_order); |
| 158 | if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { |
Yinghai Lu | 1edc1ab | 2008-04-13 01:11:41 -0700 | [diff] [blame] | 159 | printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n", |
| 160 | 32 << *order, apsizereg); |
| 161 | *order = old_order; |
| 162 | } |
| 163 | |
Ingo Molnar | 31183ba | 2008-01-30 13:30:10 +0100 | [diff] [blame] | 164 | printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n", |
| 165 | aper, 32 << *order, apsizereg); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | |
Yinghai Lu | 8c9fd91 | 2008-04-13 18:42:31 -0700 | [diff] [blame] | 167 | if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 168 | return 0; |
| 169 | return (u32)aper; |
| 170 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 171 | |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 172 | /* |
| 173 | * Look for an AGP bridge. Windows only expects the aperture in the |
| 174 | * AGP bridge and some BIOS forget to initialize the Northbridge too. |
| 175 | * Work around this here. |
| 176 | * |
| 177 | * Do an PCI bus scan by hand because we're running before the PCI |
| 178 | * subsystem. |
| 179 | * |
Hans Rosenfeld | eec1d4f | 2010-10-29 17:14:30 +0200 | [diff] [blame] | 180 | * All AMD AGP bridges are AGPv3 compliant, so we can do this scan |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 181 | * generically. It's probably overkill to always scan all slots because |
| 182 | * the AGP bridges should be always an own bus on the HT hierarchy, |
| 183 | * but do it here for future safety. |
| 184 | */ |
Pavel Machek | dd564d0 | 2008-05-27 18:03:56 +0200 | [diff] [blame] | 185 | static u32 __init search_agp_bridge(u32 *order, int *valid_agp) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 186 | { |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 187 | int bus, slot, func; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | |
| 189 | /* Poor man's PCI discovery */ |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 190 | for (bus = 0; bus < 256; bus++) { |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 191 | for (slot = 0; slot < 32; slot++) { |
| 192 | for (func = 0; func < 8; func++) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 193 | u32 class, cap; |
| 194 | u8 type; |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 195 | class = read_pci_config(bus, slot, func, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 196 | PCI_CLASS_REVISION); |
| 197 | if (class == 0xffffffff) |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 198 | break; |
| 199 | |
| 200 | switch (class >> 16) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 201 | case PCI_CLASS_BRIDGE_HOST: |
| 202 | case PCI_CLASS_BRIDGE_OTHER: /* needed? */ |
| 203 | /* AGP bridge? */ |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 204 | cap = find_cap(bus, slot, func, |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 205 | PCI_CAP_ID_AGP); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 206 | if (!cap) |
| 207 | break; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 208 | *valid_agp = 1; |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 209 | return read_agp(bus, slot, func, cap, |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 210 | order); |
| 211 | } |
| 212 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 213 | /* No multi-function device? */ |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 214 | type = read_pci_config_byte(bus, slot, func, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 215 | PCI_HEADER_TYPE); |
| 216 | if (!(type & 0x80)) |
| 217 | break; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 218 | } |
| 219 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 220 | } |
Ingo Molnar | 31183ba | 2008-01-30 13:30:10 +0100 | [diff] [blame] | 221 | printk(KERN_INFO "No AGP bridge found\n"); |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 222 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 223 | return 0; |
| 224 | } |
| 225 | |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 226 | static int gart_fix_e820 __initdata = 1; |
| 227 | |
| 228 | static int __init parse_gart_mem(char *p) |
| 229 | { |
| 230 | if (!p) |
| 231 | return -EINVAL; |
| 232 | |
| 233 | if (!strncmp(p, "off", 3)) |
| 234 | gart_fix_e820 = 0; |
| 235 | else if (!strncmp(p, "on", 2)) |
| 236 | gart_fix_e820 = 1; |
| 237 | |
| 238 | return 0; |
| 239 | } |
| 240 | early_param("gart_fix_e820", parse_gart_mem); |
| 241 | |
| 242 | void __init early_gart_iommu_check(void) |
| 243 | { |
| 244 | /* |
| 245 | * in case it is enabled before, esp for kexec/kdump, |
| 246 | * previous kernel already enable that. memset called |
| 247 | * by allocate_aperture/__alloc_bootmem_nopanic cause restart. |
| 248 | * or second kernel have different position for GART hole. and new |
| 249 | * kernel could use hole as RAM that is still used by GART set by |
| 250 | * first kernel |
| 251 | * or BIOS forget to put that in reserved. |
| 252 | * try to update e820 to make that region as reserved. |
| 253 | */ |
Andi Kleen | fa10ba6 | 2010-07-20 15:19:49 -0700 | [diff] [blame] | 254 | u32 agp_aper_order = 0; |
Yinghai Lu | f3eee54 | 2009-12-14 11:52:15 +0900 | [diff] [blame] | 255 | int i, fix, slot, valid_agp = 0; |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 256 | u32 ctl; |
| 257 | u32 aper_size = 0, aper_order = 0, last_aper_order = 0; |
| 258 | u64 aper_base = 0, last_aper_base = 0; |
Pavel Machek | fa5b8a3 | 2008-05-26 20:40:47 +0200 | [diff] [blame] | 259 | int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 260 | |
| 261 | if (!early_pci_allowed()) |
| 262 | return; |
| 263 | |
Pavel Machek | fa5b8a3 | 2008-05-26 20:40:47 +0200 | [diff] [blame] | 264 | /* This is mostly duplicate of iommu_hole_init */ |
Andi Kleen | fa10ba6 | 2010-07-20 15:19:49 -0700 | [diff] [blame] | 265 | search_agp_bridge(&agp_aper_order, &valid_agp); |
Yinghai Lu | f3eee54 | 2009-12-14 11:52:15 +0900 | [diff] [blame] | 266 | |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 267 | fix = 0; |
Jan Beulich | 24d9b70 | 2011-01-10 16:20:23 +0000 | [diff] [blame] | 268 | for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 269 | int bus; |
| 270 | int dev_base, dev_limit; |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 271 | |
Jan Beulich | 24d9b70 | 2011-01-10 16:20:23 +0000 | [diff] [blame] | 272 | bus = amd_nb_bus_dev_ranges[i].bus; |
| 273 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
| 274 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 275 | |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 276 | for (slot = dev_base; slot < dev_limit; slot++) { |
Hans Rosenfeld | eec1d4f | 2010-10-29 17:14:30 +0200 | [diff] [blame] | 277 | if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 278 | continue; |
| 279 | |
| 280 | ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); |
Borislav Petkov | 57ab43e | 2010-09-03 18:39:39 +0200 | [diff] [blame] | 281 | aper_enabled = ctl & GARTEN; |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 282 | aper_order = (ctl >> 1) & 7; |
| 283 | aper_size = (32 * 1024 * 1024) << aper_order; |
| 284 | aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; |
| 285 | aper_base <<= 25; |
| 286 | |
Pavel Machek | fa5b8a3 | 2008-05-26 20:40:47 +0200 | [diff] [blame] | 287 | if (last_valid) { |
| 288 | if ((aper_order != last_aper_order) || |
| 289 | (aper_base != last_aper_base) || |
| 290 | (aper_enabled != last_aper_enabled)) { |
| 291 | fix = 1; |
| 292 | break; |
| 293 | } |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 294 | } |
Pavel Machek | fa5b8a3 | 2008-05-26 20:40:47 +0200 | [diff] [blame] | 295 | |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 296 | last_aper_order = aper_order; |
| 297 | last_aper_base = aper_base; |
| 298 | last_aper_enabled = aper_enabled; |
Pavel Machek | fa5b8a3 | 2008-05-26 20:40:47 +0200 | [diff] [blame] | 299 | last_valid = 1; |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 300 | } |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 301 | } |
| 302 | |
| 303 | if (!fix && !aper_enabled) |
| 304 | return; |
| 305 | |
| 306 | if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) |
| 307 | fix = 1; |
| 308 | |
| 309 | if (gart_fix_e820 && !fix && aper_enabled) { |
Yinghai Lu | 0754557 | 2008-06-21 03:50:47 -0700 | [diff] [blame] | 310 | if (e820_any_mapped(aper_base, aper_base + aper_size, |
| 311 | E820_RAM)) { |
Pavel Machek | 0abbc78 | 2008-05-20 16:27:17 +0200 | [diff] [blame] | 312 | /* reserve it, so we can reuse it in second kernel */ |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 313 | printk(KERN_INFO "update e820 for GART\n"); |
Yinghai Lu | d0be6bd | 2008-06-15 18:58:51 -0700 | [diff] [blame] | 314 | e820_add_region(aper_base, aper_size, E820_RESERVED); |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 315 | update_e820(); |
| 316 | } |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 317 | } |
| 318 | |
Yinghai Lu | f3eee54 | 2009-12-14 11:52:15 +0900 | [diff] [blame] | 319 | if (valid_agp) |
Pavel Machek | 4f384f8 | 2008-05-26 21:17:30 +0200 | [diff] [blame] | 320 | return; |
| 321 | |
Yinghai Lu | f3eee54 | 2009-12-14 11:52:15 +0900 | [diff] [blame] | 322 | /* disable them all at first */ |
Jan Beulich | 24d9b70 | 2011-01-10 16:20:23 +0000 | [diff] [blame] | 323 | for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 324 | int bus; |
| 325 | int dev_base, dev_limit; |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 326 | |
Jan Beulich | 24d9b70 | 2011-01-10 16:20:23 +0000 | [diff] [blame] | 327 | bus = amd_nb_bus_dev_ranges[i].bus; |
| 328 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
| 329 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 330 | |
| 331 | for (slot = dev_base; slot < dev_limit; slot++) { |
Hans Rosenfeld | eec1d4f | 2010-10-29 17:14:30 +0200 | [diff] [blame] | 332 | if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 333 | continue; |
| 334 | |
| 335 | ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); |
Borislav Petkov | 57ab43e | 2010-09-03 18:39:39 +0200 | [diff] [blame] | 336 | ctl &= ~GARTEN; |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 337 | write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); |
| 338 | } |
Yinghai Lu | aaf2304 | 2008-01-30 13:33:09 +0100 | [diff] [blame] | 339 | } |
| 340 | |
| 341 | } |
| 342 | |
Yinghai Lu | 8c9fd91 | 2008-04-13 18:42:31 -0700 | [diff] [blame] | 343 | static int __initdata printed_gart_size_msg; |
| 344 | |
Konrad Rzeszutek Wilk | 480125b | 2010-08-26 13:57:57 -0400 | [diff] [blame] | 345 | int __init gart_iommu_hole_init(void) |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 346 | { |
Yinghai Lu | 8c9fd91 | 2008-04-13 18:42:31 -0700 | [diff] [blame] | 347 | u32 agp_aper_base = 0, agp_aper_order = 0; |
Andi Kleen | 50895c5 | 2005-11-05 17:25:53 +0100 | [diff] [blame] | 348 | u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 349 | u64 aper_base, last_aper_base = 0; |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 350 | int fix, slot, valid_agp = 0; |
| 351 | int i, node; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 352 | |
Joerg Roedel | 0440d4c | 2007-10-24 12:49:50 +0200 | [diff] [blame] | 353 | if (gart_iommu_aperture_disabled || !fix_aperture || |
| 354 | !early_pci_allowed()) |
Konrad Rzeszutek Wilk | 480125b | 2010-08-26 13:57:57 -0400 | [diff] [blame] | 355 | return -ENODEV; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 356 | |
Dan Aloni | 753811d | 2007-07-21 17:11:36 +0200 | [diff] [blame] | 357 | printk(KERN_INFO "Checking aperture...\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 358 | |
Yinghai Lu | 8c9fd91 | 2008-04-13 18:42:31 -0700 | [diff] [blame] | 359 | if (!fallback_aper_force) |
| 360 | agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); |
| 361 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 362 | fix = 0; |
Yinghai Lu | 47db4c3 | 2008-01-30 13:33:18 +0100 | [diff] [blame] | 363 | node = 0; |
Jan Beulich | 24d9b70 | 2011-01-10 16:20:23 +0000 | [diff] [blame] | 364 | for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 365 | int bus; |
| 366 | int dev_base, dev_limit; |
Joerg Roedel | 4b83873 | 2010-04-07 12:57:35 +0200 | [diff] [blame] | 367 | u32 ctl; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 368 | |
Jan Beulich | 24d9b70 | 2011-01-10 16:20:23 +0000 | [diff] [blame] | 369 | bus = amd_nb_bus_dev_ranges[i].bus; |
| 370 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
| 371 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 372 | |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 373 | for (slot = dev_base; slot < dev_limit; slot++) { |
Hans Rosenfeld | eec1d4f | 2010-10-29 17:14:30 +0200 | [diff] [blame] | 374 | if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 375 | continue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 376 | |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 377 | iommu_detected = 1; |
| 378 | gart_iommu_aperture = 1; |
FUJITA Tomonori | de95762 | 2009-11-10 19:46:14 +0900 | [diff] [blame] | 379 | x86_init.iommu.iommu_init = gart_iommu_init; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 380 | |
Joerg Roedel | 4b83873 | 2010-04-07 12:57:35 +0200 | [diff] [blame] | 381 | ctl = read_pci_config(bus, slot, 3, |
| 382 | AMD64_GARTAPERTURECTL); |
| 383 | |
| 384 | /* |
| 385 | * Before we do anything else disable the GART. It may |
| 386 | * still be enabled if we boot into a crash-kernel here. |
| 387 | * Reconfiguring the GART while it is enabled could have |
| 388 | * unknown side-effects. |
| 389 | */ |
| 390 | ctl &= ~GARTEN; |
| 391 | write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); |
| 392 | |
| 393 | aper_order = (ctl >> 1) & 7; |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 394 | aper_size = (32 * 1024 * 1024) << aper_order; |
| 395 | aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; |
| 396 | aper_base <<= 25; |
| 397 | |
| 398 | printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n", |
| 399 | node, aper_base, aper_size >> 20); |
| 400 | node++; |
| 401 | |
| 402 | if (!aperture_valid(aper_base, aper_size, 64<<20)) { |
| 403 | if (valid_agp && agp_aper_base && |
| 404 | agp_aper_base == aper_base && |
| 405 | agp_aper_order == aper_order) { |
| 406 | /* the same between two setting from NB and agp */ |
Yinghai Lu | c987d12 | 2008-06-24 22:14:09 -0700 | [diff] [blame] | 407 | if (!no_iommu && |
| 408 | max_pfn > MAX_DMA32_PFN && |
| 409 | !printed_gart_size_msg) { |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 410 | printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n"); |
| 411 | printk(KERN_ERR "please increase GART size in your BIOS setup\n"); |
| 412 | printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n"); |
| 413 | printed_gart_size_msg = 1; |
| 414 | } |
| 415 | } else { |
| 416 | fix = 1; |
| 417 | goto out; |
Yinghai Lu | 8c9fd91 | 2008-04-13 18:42:31 -0700 | [diff] [blame] | 418 | } |
Yinghai Lu | 8c9fd91 | 2008-04-13 18:42:31 -0700 | [diff] [blame] | 419 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 420 | |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 421 | if ((last_aper_order && aper_order != last_aper_order) || |
| 422 | (last_aper_base && aper_base != last_aper_base)) { |
| 423 | fix = 1; |
| 424 | goto out; |
| 425 | } |
| 426 | last_aper_order = aper_order; |
| 427 | last_aper_base = aper_base; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 428 | } |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 429 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 430 | |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 431 | out: |
Aaron Durbin | 56dd669 | 2006-09-26 10:52:40 +0200 | [diff] [blame] | 432 | if (!fix && !fallback_aper_force) { |
Bjorn Helgaas | 707d4ee | 2014-03-18 14:26:12 -0600 | [diff] [blame] | 433 | if (last_aper_base) |
Konrad Rzeszutek Wilk | 480125b | 2010-08-26 13:57:57 -0400 | [diff] [blame] | 434 | return 1; |
Konrad Rzeszutek Wilk | 480125b | 2010-08-26 13:57:57 -0400 | [diff] [blame] | 435 | return 0; |
Aaron Durbin | 56dd669 | 2006-09-26 10:52:40 +0200 | [diff] [blame] | 436 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 437 | |
Yinghai Lu | 8c9fd91 | 2008-04-13 18:42:31 -0700 | [diff] [blame] | 438 | if (!fallback_aper_force) { |
| 439 | aper_alloc = agp_aper_base; |
| 440 | aper_order = agp_aper_order; |
| 441 | } |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 442 | |
| 443 | if (aper_alloc) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 444 | /* Got the aperture from the AGP bridge */ |
Yinghai Lu | c987d12 | 2008-06-24 22:14:09 -0700 | [diff] [blame] | 445 | } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 446 | force_iommu || |
| 447 | valid_agp || |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 448 | fallback_aper_force) { |
Adam Jackson | 9b15684 | 2008-09-29 14:52:03 -0400 | [diff] [blame] | 449 | printk(KERN_INFO |
Ingo Molnar | 31183ba | 2008-01-30 13:30:10 +0100 | [diff] [blame] | 450 | "Your BIOS doesn't leave a aperture memory hole\n"); |
Adam Jackson | 9b15684 | 2008-09-29 14:52:03 -0400 | [diff] [blame] | 451 | printk(KERN_INFO |
Ingo Molnar | 31183ba | 2008-01-30 13:30:10 +0100 | [diff] [blame] | 452 | "Please enable the IOMMU option in the BIOS setup\n"); |
Adam Jackson | 9b15684 | 2008-09-29 14:52:03 -0400 | [diff] [blame] | 453 | printk(KERN_INFO |
Ingo Molnar | 31183ba | 2008-01-30 13:30:10 +0100 | [diff] [blame] | 454 | "This costs you %d MB of RAM\n", |
| 455 | 32 << fallback_aper_order); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 456 | |
| 457 | aper_order = fallback_aper_order; |
| 458 | aper_alloc = allocate_aperture(); |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 459 | if (!aper_alloc) { |
| 460 | /* |
| 461 | * Could disable AGP and IOMMU here, but it's |
| 462 | * probably not worth it. But the later users |
| 463 | * cannot deal with bad apertures and turning |
| 464 | * on the aperture over memory causes very |
| 465 | * strange problems, so it's better to panic |
| 466 | * early. |
| 467 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 468 | panic("Not enough memory for aperture"); |
| 469 | } |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 470 | } else { |
Konrad Rzeszutek Wilk | 480125b | 2010-08-26 13:57:57 -0400 | [diff] [blame] | 471 | return 0; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 472 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 473 | |
| 474 | /* Fix up the north bridges */ |
Jan Beulich | 24d9b70 | 2011-01-10 16:20:23 +0000 | [diff] [blame] | 475 | for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
Borislav Petkov | 260133a | 2010-09-03 18:39:40 +0200 | [diff] [blame] | 476 | int bus, dev_base, dev_limit; |
| 477 | |
| 478 | /* |
| 479 | * Don't enable translation yet but enable GART IO and CPU |
| 480 | * accesses and set DISTLBWALKPRB since GART table memory is UC. |
| 481 | */ |
Joerg Roedel | c34151a | 2011-04-18 15:45:45 +0200 | [diff] [blame] | 482 | u32 ctl = aper_order << 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 483 | |
Jan Beulich | 24d9b70 | 2011-01-10 16:20:23 +0000 | [diff] [blame] | 484 | bus = amd_nb_bus_dev_ranges[i].bus; |
| 485 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
| 486 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 487 | for (slot = dev_base; slot < dev_limit; slot++) { |
Hans Rosenfeld | eec1d4f | 2010-10-29 17:14:30 +0200 | [diff] [blame] | 488 | if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 489 | continue; |
| 490 | |
Borislav Petkov | 260133a | 2010-09-03 18:39:40 +0200 | [diff] [blame] | 491 | write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); |
Yinghai Lu | 55c0d72 | 2008-04-19 01:31:11 -0700 | [diff] [blame] | 492 | write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); |
| 493 | } |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 494 | } |
Rafael J. Wysocki | 6703f6d | 2008-06-10 00:10:48 +0200 | [diff] [blame] | 495 | |
| 496 | set_up_gart_resume(aper_order, aper_alloc); |
Konrad Rzeszutek Wilk | 480125b | 2010-08-26 13:57:57 -0400 | [diff] [blame] | 497 | |
| 498 | return 1; |
Ingo Molnar | c140df9 | 2008-01-30 13:30:09 +0100 | [diff] [blame] | 499 | } |