blob: 416f6ac65b761080a2e81a8ace5c5d442a8ea2d1 [file] [log] [blame]
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -07001/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
mark gross98bcef52008-02-23 15:23:35 -080017 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070021 *
Suresh Siddhae61d98d2008-07-10 11:16:35 -070022 * This file implements early detection/parsing of Remapping Devices
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070023 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
Suresh Siddhae61d98d2008-07-10 11:16:35 -070025 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070027 */
28
29#include <linux/pci.h>
30#include <linux/dmar.h>
Kay, Allen M38717942008-09-09 18:37:29 +030031#include <linux/iova.h>
32#include <linux/intel-iommu.h>
Suresh Siddhafe962e92008-07-10 11:16:42 -070033#include <linux/timer.h>
Suresh Siddha0ac24912009-03-16 17:04:54 -070034#include <linux/irq.h>
35#include <linux/interrupt.h>
Shane Wang69575d32009-09-01 18:25:07 -070036#include <linux/tboot.h>
Len Browneb27cae2009-07-06 23:40:19 -040037#include <linux/dmi.h>
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070038
Len Browna192a952009-07-28 16:45:54 -040039#define PREFIX "DMAR: "
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070040
41/* No locks are needed as DMA remapping hardware unit
42 * list is constructed at boot time and hotplug of
43 * these units are not supported by the architecture.
44 */
45LIST_HEAD(dmar_drhd_units);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070046
47static struct acpi_table_header * __initdata dmar_tbl;
Yinghai Lu8e1568f2009-02-11 01:06:59 -080048static acpi_size dmar_tbl_size;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070049
50static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
51{
52 /*
53 * add INCLUDE_ALL at the tail, so scan the list will find it at
54 * the very end.
55 */
56 if (drhd->include_all)
57 list_add_tail(&drhd->list, &dmar_drhd_units);
58 else
59 list_add(&drhd->list, &dmar_drhd_units);
60}
61
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070062static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
63 struct pci_dev **dev, u16 segment)
64{
65 struct pci_bus *bus;
66 struct pci_dev *pdev = NULL;
67 struct acpi_dmar_pci_path *path;
68 int count;
69
70 bus = pci_find_bus(segment, scope->bus);
71 path = (struct acpi_dmar_pci_path *)(scope + 1);
72 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
73 / sizeof(struct acpi_dmar_pci_path);
74
75 while (count) {
76 if (pdev)
77 pci_dev_put(pdev);
78 /*
79 * Some BIOSes list non-exist devices in DMAR table, just
80 * ignore it
81 */
82 if (!bus) {
83 printk(KERN_WARNING
84 PREFIX "Device scope bus [%d] not found\n",
85 scope->bus);
86 break;
87 }
88 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
89 if (!pdev) {
90 printk(KERN_WARNING PREFIX
91 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
92 segment, bus->number, path->dev, path->fn);
93 break;
94 }
95 path ++;
96 count --;
97 bus = pdev->subordinate;
98 }
99 if (!pdev) {
100 printk(KERN_WARNING PREFIX
101 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
102 segment, scope->bus, path->dev, path->fn);
103 *dev = NULL;
104 return 0;
105 }
106 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
107 pdev->subordinate) || (scope->entry_type == \
108 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
109 pci_dev_put(pdev);
110 printk(KERN_WARNING PREFIX
111 "Device scope type does not match for %s\n",
112 pci_name(pdev));
113 return -EINVAL;
114 }
115 *dev = pdev;
116 return 0;
117}
118
119static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
120 struct pci_dev ***devices, u16 segment)
121{
122 struct acpi_dmar_device_scope *scope;
123 void * tmp = start;
124 int index;
125 int ret;
126
127 *cnt = 0;
128 while (start < end) {
129 scope = start;
130 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
131 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
132 (*cnt)++;
133 else
134 printk(KERN_WARNING PREFIX
135 "Unsupported device scope\n");
136 start += scope->length;
137 }
138 if (*cnt == 0)
139 return 0;
140
141 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
142 if (!*devices)
143 return -ENOMEM;
144
145 start = tmp;
146 index = 0;
147 while (start < end) {
148 scope = start;
149 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
150 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
151 ret = dmar_parse_one_dev_scope(scope,
152 &(*devices)[index], segment);
153 if (ret) {
154 kfree(*devices);
155 return ret;
156 }
157 index ++;
158 }
159 start += scope->length;
160 }
161
162 return 0;
163}
164
165/**
166 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
167 * structure which uniquely represent one DMA remapping hardware unit
168 * present in the platform
169 */
170static int __init
171dmar_parse_one_drhd(struct acpi_dmar_header *header)
172{
173 struct acpi_dmar_hardware_unit *drhd;
174 struct dmar_drhd_unit *dmaru;
175 int ret = 0;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700176
David Woodhousee523b382009-04-10 22:27:48 -0700177 drhd = (struct acpi_dmar_hardware_unit *)header;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700178 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
179 if (!dmaru)
180 return -ENOMEM;
181
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700182 dmaru->hdr = header;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700183 dmaru->reg_base_addr = drhd->address;
David Woodhouse276dbf992009-04-04 01:45:37 +0100184 dmaru->segment = drhd->segment;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700185 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
186
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700187 ret = alloc_iommu(dmaru);
188 if (ret) {
189 kfree(dmaru);
190 return ret;
191 }
192 dmar_register_drhd_unit(dmaru);
193 return 0;
194}
195
David Woodhousef82851a2008-10-18 15:43:14 +0100196static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700197{
198 struct acpi_dmar_hardware_unit *drhd;
David Woodhousef82851a2008-10-18 15:43:14 +0100199 int ret = 0;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700200
201 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
202
Yu Zhao2e824f72008-12-22 16:54:58 +0800203 if (dmaru->include_all)
204 return 0;
205
206 ret = dmar_parse_dev_scope((void *)(drhd + 1),
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700207 ((void *)drhd) + drhd->header.length,
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700208 &dmaru->devices_cnt, &dmaru->devices,
209 drhd->segment);
Suresh Siddha1c7d1bc2008-09-03 16:58:35 -0700210 if (ret) {
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700211 list_del(&dmaru->list);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700212 kfree(dmaru);
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700213 }
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700214 return ret;
215}
216
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700217#ifdef CONFIG_DMAR
218LIST_HEAD(dmar_rmrr_units);
219
220static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
221{
222 list_add(&rmrr->list, &dmar_rmrr_units);
223}
224
225
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700226static int __init
227dmar_parse_one_rmrr(struct acpi_dmar_header *header)
228{
229 struct acpi_dmar_reserved_memory *rmrr;
230 struct dmar_rmrr_unit *rmrru;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700231
232 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
233 if (!rmrru)
234 return -ENOMEM;
235
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700236 rmrru->hdr = header;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700237 rmrr = (struct acpi_dmar_reserved_memory *)header;
238 rmrru->base_address = rmrr->base_address;
239 rmrru->end_address = rmrr->end_address;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700240
241 dmar_register_rmrr_unit(rmrru);
242 return 0;
243}
244
245static int __init
246rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
247{
248 struct acpi_dmar_reserved_memory *rmrr;
249 int ret;
250
251 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700252 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700253 ((void *)rmrr) + rmrr->header.length,
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700254 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
255
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700256 if (ret || (rmrru->devices_cnt == 0)) {
257 list_del(&rmrru->list);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700258 kfree(rmrru);
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700259 }
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700260 return ret;
261}
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800262
263static LIST_HEAD(dmar_atsr_units);
264
265static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
266{
267 struct acpi_dmar_atsr *atsr;
268 struct dmar_atsr_unit *atsru;
269
270 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
271 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
272 if (!atsru)
273 return -ENOMEM;
274
275 atsru->hdr = hdr;
276 atsru->include_all = atsr->flags & 0x1;
277
278 list_add(&atsru->list, &dmar_atsr_units);
279
280 return 0;
281}
282
283static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
284{
285 int rc;
286 struct acpi_dmar_atsr *atsr;
287
288 if (atsru->include_all)
289 return 0;
290
291 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
292 rc = dmar_parse_dev_scope((void *)(atsr + 1),
293 (void *)atsr + atsr->header.length,
294 &atsru->devices_cnt, &atsru->devices,
295 atsr->segment);
296 if (rc || !atsru->devices_cnt) {
297 list_del(&atsru->list);
298 kfree(atsru);
299 }
300
301 return rc;
302}
303
304int dmar_find_matched_atsr_unit(struct pci_dev *dev)
305{
306 int i;
307 struct pci_bus *bus;
308 struct acpi_dmar_atsr *atsr;
309 struct dmar_atsr_unit *atsru;
310
311 list_for_each_entry(atsru, &dmar_atsr_units, list) {
312 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
313 if (atsr->segment == pci_domain_nr(dev->bus))
314 goto found;
315 }
316
317 return 0;
318
319found:
320 for (bus = dev->bus; bus; bus = bus->parent) {
321 struct pci_dev *bridge = bus->self;
322
323 if (!bridge || !bridge->is_pcie ||
324 bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
325 return 0;
326
327 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
328 for (i = 0; i < atsru->devices_cnt; i++)
329 if (atsru->devices[i] == bridge)
330 return 1;
331 break;
332 }
333 }
334
335 if (atsru->include_all)
336 return 1;
337
338 return 0;
339}
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700340#endif
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700341
342static void __init
343dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
344{
345 struct acpi_dmar_hardware_unit *drhd;
346 struct acpi_dmar_reserved_memory *rmrr;
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800347 struct acpi_dmar_atsr *atsr;
Roland Dreier17b60972009-09-24 12:14:00 -0700348 struct acpi_dmar_rhsa *rhsa;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700349
350 switch (header->type) {
351 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800352 drhd = container_of(header, struct acpi_dmar_hardware_unit,
353 header);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700354 printk (KERN_INFO PREFIX
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800355 "DRHD base: %#016Lx flags: %#x\n",
356 (unsigned long long)drhd->address, drhd->flags);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700357 break;
358 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800359 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
360 header);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700361 printk (KERN_INFO PREFIX
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800362 "RMRR base: %#016Lx end: %#016Lx\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700363 (unsigned long long)rmrr->base_address,
364 (unsigned long long)rmrr->end_address);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700365 break;
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800366 case ACPI_DMAR_TYPE_ATSR:
367 atsr = container_of(header, struct acpi_dmar_atsr, header);
368 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
369 break;
Roland Dreier17b60972009-09-24 12:14:00 -0700370 case ACPI_DMAR_HARDWARE_AFFINITY:
371 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
372 printk(KERN_INFO PREFIX "RHSA base: %#016Lx proximity domain: %#x\n",
373 (unsigned long long)rhsa->base_address,
374 rhsa->proximity_domain);
375 break;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700376 }
377}
378
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700379/**
380 * dmar_table_detect - checks to see if the platform supports DMAR devices
381 */
382static int __init dmar_table_detect(void)
383{
384 acpi_status status = AE_OK;
385
386 /* if we could find DMAR table, then there are DMAR devices */
Yinghai Lu8e1568f2009-02-11 01:06:59 -0800387 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
388 (struct acpi_table_header **)&dmar_tbl,
389 &dmar_tbl_size);
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700390
391 if (ACPI_SUCCESS(status) && !dmar_tbl) {
392 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
393 status = AE_NOT_FOUND;
394 }
395
396 return (ACPI_SUCCESS(status) ? 1 : 0);
397}
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700398
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700399/**
400 * parse_dmar_table - parses the DMA reporting table
401 */
402static int __init
403parse_dmar_table(void)
404{
405 struct acpi_table_dmar *dmar;
406 struct acpi_dmar_header *entry_header;
407 int ret = 0;
408
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700409 /*
410 * Do it again, earlier dmar_tbl mapping could be mapped with
411 * fixed map.
412 */
413 dmar_table_detect();
414
Joseph Cihulaa59b50e2009-06-30 19:31:10 -0700415 /*
416 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
417 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
418 */
419 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
420
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700421 dmar = (struct acpi_table_dmar *)dmar_tbl;
422 if (!dmar)
423 return -ENODEV;
424
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700425 if (dmar->width < PAGE_SHIFT - 1) {
Fenghua Yu093f87d2007-11-21 15:07:14 -0800426 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700427 return -EINVAL;
428 }
429
430 printk (KERN_INFO PREFIX "Host address width %d\n",
431 dmar->width + 1);
432
433 entry_header = (struct acpi_dmar_header *)(dmar + 1);
434 while (((unsigned long)entry_header) <
435 (((unsigned long)dmar) + dmar_tbl->length)) {
Tony Battersby084eb962009-02-11 13:24:19 -0800436 /* Avoid looping forever on bad ACPI tables */
437 if (entry_header->length == 0) {
438 printk(KERN_WARNING PREFIX
439 "Invalid 0-length structure\n");
440 ret = -EINVAL;
441 break;
442 }
443
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700444 dmar_table_print_dmar_entry(entry_header);
445
446 switch (entry_header->type) {
447 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
448 ret = dmar_parse_one_drhd(entry_header);
449 break;
450 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700451#ifdef CONFIG_DMAR
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700452 ret = dmar_parse_one_rmrr(entry_header);
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700453#endif
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700454 break;
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800455 case ACPI_DMAR_TYPE_ATSR:
456#ifdef CONFIG_DMAR
457 ret = dmar_parse_one_atsr(entry_header);
458#endif
459 break;
Roland Dreier17b60972009-09-24 12:14:00 -0700460 case ACPI_DMAR_HARDWARE_AFFINITY:
461 /* We don't do anything with RHSA (yet?) */
462 break;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700463 default:
464 printk(KERN_WARNING PREFIX
Roland Dreier4de75cf2009-09-24 01:01:29 +0100465 "Unknown DMAR structure type %d\n",
466 entry_header->type);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700467 ret = 0; /* for forward compatibility */
468 break;
469 }
470 if (ret)
471 break;
472
473 entry_header = ((void *)entry_header + entry_header->length);
474 }
475 return ret;
476}
477
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700478int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
479 struct pci_dev *dev)
480{
481 int index;
482
483 while (dev) {
484 for (index = 0; index < cnt; index++)
485 if (dev == devices[index])
486 return 1;
487
488 /* Check our parent */
489 dev = dev->bus->self;
490 }
491
492 return 0;
493}
494
495struct dmar_drhd_unit *
496dmar_find_matched_drhd_unit(struct pci_dev *dev)
497{
Yu Zhao2e824f72008-12-22 16:54:58 +0800498 struct dmar_drhd_unit *dmaru = NULL;
499 struct acpi_dmar_hardware_unit *drhd;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700500
Yu Zhao2e824f72008-12-22 16:54:58 +0800501 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
502 drhd = container_of(dmaru->hdr,
503 struct acpi_dmar_hardware_unit,
504 header);
505
506 if (dmaru->include_all &&
507 drhd->segment == pci_domain_nr(dev->bus))
508 return dmaru;
509
510 if (dmar_pci_device_match(dmaru->devices,
511 dmaru->devices_cnt, dev))
512 return dmaru;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700513 }
514
515 return NULL;
516}
517
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700518int __init dmar_dev_scope_init(void)
519{
Suresh Siddha04e2ea62008-09-03 16:58:34 -0700520 struct dmar_drhd_unit *drhd, *drhd_n;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700521 int ret = -ENODEV;
522
Suresh Siddha04e2ea62008-09-03 16:58:34 -0700523 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700524 ret = dmar_parse_dev(drhd);
525 if (ret)
526 return ret;
527 }
528
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700529#ifdef CONFIG_DMAR
530 {
Suresh Siddha04e2ea62008-09-03 16:58:34 -0700531 struct dmar_rmrr_unit *rmrr, *rmrr_n;
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800532 struct dmar_atsr_unit *atsr, *atsr_n;
533
Suresh Siddha04e2ea62008-09-03 16:58:34 -0700534 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700535 ret = rmrr_parse_dev(rmrr);
536 if (ret)
537 return ret;
538 }
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800539
540 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
541 ret = atsr_parse_dev(atsr);
542 if (ret)
543 return ret;
544 }
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700545 }
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700546#endif
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700547
548 return ret;
549}
550
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700551
552int __init dmar_table_init(void)
553{
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700554 static int dmar_table_initialized;
Fenghua Yu093f87d2007-11-21 15:07:14 -0800555 int ret;
556
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700557 if (dmar_table_initialized)
558 return 0;
559
560 dmar_table_initialized = 1;
561
Fenghua Yu093f87d2007-11-21 15:07:14 -0800562 ret = parse_dmar_table();
563 if (ret) {
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700564 if (ret != -ENODEV)
565 printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
Fenghua Yu093f87d2007-11-21 15:07:14 -0800566 return ret;
567 }
568
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700569 if (list_empty(&dmar_drhd_units)) {
570 printk(KERN_INFO PREFIX "No DMAR devices found\n");
571 return -ENODEV;
572 }
Fenghua Yu093f87d2007-11-21 15:07:14 -0800573
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700574#ifdef CONFIG_DMAR
Suresh Siddha2d6b5f82008-07-10 11:16:39 -0700575 if (list_empty(&dmar_rmrr_units))
Fenghua Yu093f87d2007-11-21 15:07:14 -0800576 printk(KERN_INFO PREFIX "No RMRR found\n");
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800577
578 if (list_empty(&dmar_atsr_units))
579 printk(KERN_INFO PREFIX "No ATSR found\n");
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700580#endif
Fenghua Yu093f87d2007-11-21 15:07:14 -0800581
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700582 return 0;
583}
584
David Woodhouse86cf8982009-11-09 22:15:15 +0000585int __init check_zero_address(void)
586{
587 struct acpi_table_dmar *dmar;
588 struct acpi_dmar_header *entry_header;
589 struct acpi_dmar_hardware_unit *drhd;
590
591 dmar = (struct acpi_table_dmar *)dmar_tbl;
592 entry_header = (struct acpi_dmar_header *)(dmar + 1);
593
594 while (((unsigned long)entry_header) <
595 (((unsigned long)dmar) + dmar_tbl->length)) {
596 /* Avoid looping forever on bad ACPI tables */
597 if (entry_header->length == 0) {
598 printk(KERN_WARNING PREFIX
599 "Invalid 0-length structure\n");
600 return 0;
601 }
602
603 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
604 drhd = (void *)entry_header;
605 if (!drhd->address) {
606 /* Promote an attitude of violence to a BIOS engineer today */
607 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
608 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
609 dmi_get_system_info(DMI_BIOS_VENDOR),
610 dmi_get_system_info(DMI_BIOS_VERSION),
611 dmi_get_system_info(DMI_PRODUCT_VERSION));
David Woodhouse5854d9c2009-11-19 02:18:44 +0000612#ifdef CONFIG_DMAR
613 dmar_disabled = 1;
614#endif
David Woodhouse86cf8982009-11-09 22:15:15 +0000615 return 0;
616 }
617 break;
618 }
619
620 entry_header = ((void *)entry_header + entry_header->length);
621 }
622 return 1;
623}
624
Suresh Siddha2ae21012008-07-10 11:16:43 -0700625void __init detect_intel_iommu(void)
626{
627 int ret;
628
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700629 ret = dmar_table_detect();
David Woodhouse86cf8982009-11-09 22:15:15 +0000630 if (ret)
631 ret = check_zero_address();
Suresh Siddha2ae21012008-07-10 11:16:43 -0700632 {
Youquan Songcacd4212008-10-16 16:31:57 -0700633#ifdef CONFIG_INTR_REMAP
Suresh Siddha1cb11582008-07-10 11:16:51 -0700634 struct acpi_table_dmar *dmar;
635 /*
636 * for now we will disable dma-remapping when interrupt
637 * remapping is enabled.
638 * When support for queued invalidation for IOTLB invalidation
639 * is added, we will not need this any more.
640 */
641 dmar = (struct acpi_table_dmar *) dmar_tbl;
Youquan Songcacd4212008-10-16 16:31:57 -0700642 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
Suresh Siddha1cb11582008-07-10 11:16:51 -0700643 printk(KERN_INFO
644 "Queued invalidation will be enabled to support "
645 "x2apic and Intr-remapping.\n");
Youquan Songcacd4212008-10-16 16:31:57 -0700646#endif
Youquan Songcacd4212008-10-16 16:31:57 -0700647#ifdef CONFIG_DMAR
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +0900648 if (ret && !no_iommu && !iommu_detected && !dmar_disabled)
Suresh Siddha2ae21012008-07-10 11:16:43 -0700649 iommu_detected = 1;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700650#endif
FUJITA Tomonori9d5ce732009-11-10 19:46:16 +0900651#ifdef CONFIG_X86
652 if (ret)
653 x86_init.iommu.iommu_init = intel_iommu_init;
654#endif
Youquan Songcacd4212008-10-16 16:31:57 -0700655 }
Yinghai Lu8e1568f2009-02-11 01:06:59 -0800656 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700657 dmar_tbl = NULL;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700658}
659
660
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700661int alloc_iommu(struct dmar_drhd_unit *drhd)
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700662{
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700663 struct intel_iommu *iommu;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700664 int map_size;
665 u32 ver;
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700666 static int iommu_allocated = 0;
Joerg Roedel43f73922009-01-03 23:56:27 +0100667 int agaw = 0;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700668 int msagaw = 0;
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700669
670 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
671 if (!iommu)
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700672 return -ENOMEM;
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700673
674 iommu->seq_id = iommu_allocated++;
Suresh Siddha9d783ba2009-03-16 17:04:55 -0700675 sprintf (iommu->name, "dmar%d", iommu->seq_id);
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700676
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700677 iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700678 if (!iommu->reg) {
679 printk(KERN_ERR "IOMMU: can't map the region\n");
680 goto error;
681 }
682 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
683 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
684
David Woodhouse08155652009-08-04 09:17:20 +0100685 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
686 /* Promote an attitude of violence to a BIOS engineer today */
687 WARN(1, "Your BIOS is broken; DMAR reported at address %llx returns all ones!\n"
688 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
689 drhd->reg_base_addr,
690 dmi_get_system_info(DMI_BIOS_VENDOR),
691 dmi_get_system_info(DMI_BIOS_VERSION),
692 dmi_get_system_info(DMI_PRODUCT_VERSION));
693 goto err_unmap;
694 }
695
Joerg Roedel43f73922009-01-03 23:56:27 +0100696#ifdef CONFIG_DMAR
Weidong Han1b573682008-12-08 15:34:06 +0800697 agaw = iommu_calculate_agaw(iommu);
698 if (agaw < 0) {
699 printk(KERN_ERR
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700700 "Cannot get a valid agaw for iommu (seq_id = %d)\n",
701 iommu->seq_id);
David Woodhouse08155652009-08-04 09:17:20 +0100702 goto err_unmap;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700703 }
704 msagaw = iommu_calculate_max_sagaw(iommu);
705 if (msagaw < 0) {
706 printk(KERN_ERR
707 "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
Weidong Han1b573682008-12-08 15:34:06 +0800708 iommu->seq_id);
David Woodhouse08155652009-08-04 09:17:20 +0100709 goto err_unmap;
Weidong Han1b573682008-12-08 15:34:06 +0800710 }
Joerg Roedel43f73922009-01-03 23:56:27 +0100711#endif
Weidong Han1b573682008-12-08 15:34:06 +0800712 iommu->agaw = agaw;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700713 iommu->msagaw = msagaw;
Weidong Han1b573682008-12-08 15:34:06 +0800714
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700715 /* the registers might be more than one page */
716 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
717 cap_max_fault_reg_offset(iommu->cap));
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700718 map_size = VTD_PAGE_ALIGN(map_size);
719 if (map_size > VTD_PAGE_SIZE) {
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700720 iounmap(iommu->reg);
721 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
722 if (!iommu->reg) {
723 printk(KERN_ERR "IOMMU: can't map the region\n");
724 goto error;
725 }
726 }
727
728 ver = readl(iommu->reg + DMAR_VER_REG);
David Woodhouse08155652009-08-04 09:17:20 +0100729 pr_info("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700730 (unsigned long long)drhd->reg_base_addr,
731 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
732 (unsigned long long)iommu->cap,
733 (unsigned long long)iommu->ecap);
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700734
735 spin_lock_init(&iommu->register_lock);
736
737 drhd->iommu = iommu;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700738 return 0;
David Woodhouse08155652009-08-04 09:17:20 +0100739
740 err_unmap:
741 iounmap(iommu->reg);
742 error:
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700743 kfree(iommu);
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700744 return -1;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700745}
746
747void free_iommu(struct intel_iommu *iommu)
748{
749 if (!iommu)
750 return;
751
752#ifdef CONFIG_DMAR
753 free_dmar_iommu(iommu);
754#endif
755
756 if (iommu->reg)
757 iounmap(iommu->reg);
758 kfree(iommu);
759}
Suresh Siddhafe962e92008-07-10 11:16:42 -0700760
761/*
762 * Reclaim all the submitted descriptors which have completed its work.
763 */
764static inline void reclaim_free_desc(struct q_inval *qi)
765{
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800766 while (qi->desc_status[qi->free_tail] == QI_DONE ||
767 qi->desc_status[qi->free_tail] == QI_ABORT) {
Suresh Siddhafe962e92008-07-10 11:16:42 -0700768 qi->desc_status[qi->free_tail] = QI_FREE;
769 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
770 qi->free_cnt++;
771 }
772}
773
Yu Zhao704126a2009-01-04 16:28:52 +0800774static int qi_check_fault(struct intel_iommu *iommu, int index)
775{
776 u32 fault;
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800777 int head, tail;
Yu Zhao704126a2009-01-04 16:28:52 +0800778 struct q_inval *qi = iommu->qi;
779 int wait_index = (index + 1) % QI_LENGTH;
780
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800781 if (qi->desc_status[wait_index] == QI_ABORT)
782 return -EAGAIN;
783
Yu Zhao704126a2009-01-04 16:28:52 +0800784 fault = readl(iommu->reg + DMAR_FSTS_REG);
785
786 /*
787 * If IQE happens, the head points to the descriptor associated
788 * with the error. No new descriptors are fetched until the IQE
789 * is cleared.
790 */
791 if (fault & DMA_FSTS_IQE) {
792 head = readl(iommu->reg + DMAR_IQH_REG);
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800793 if ((head >> DMAR_IQ_SHIFT) == index) {
794 printk(KERN_ERR "VT-d detected invalid descriptor: "
795 "low=%llx, high=%llx\n",
796 (unsigned long long)qi->desc[index].low,
797 (unsigned long long)qi->desc[index].high);
Yu Zhao704126a2009-01-04 16:28:52 +0800798 memcpy(&qi->desc[index], &qi->desc[wait_index],
799 sizeof(struct qi_desc));
800 __iommu_flush_cache(iommu, &qi->desc[index],
801 sizeof(struct qi_desc));
802 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
803 return -EINVAL;
804 }
805 }
806
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800807 /*
808 * If ITE happens, all pending wait_desc commands are aborted.
809 * No new descriptors are fetched until the ITE is cleared.
810 */
811 if (fault & DMA_FSTS_ITE) {
812 head = readl(iommu->reg + DMAR_IQH_REG);
813 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
814 head |= 1;
815 tail = readl(iommu->reg + DMAR_IQT_REG);
816 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
817
818 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
819
820 do {
821 if (qi->desc_status[head] == QI_IN_USE)
822 qi->desc_status[head] = QI_ABORT;
823 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
824 } while (head != tail);
825
826 if (qi->desc_status[wait_index] == QI_ABORT)
827 return -EAGAIN;
828 }
829
830 if (fault & DMA_FSTS_ICE)
831 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
832
Yu Zhao704126a2009-01-04 16:28:52 +0800833 return 0;
834}
835
Suresh Siddhafe962e92008-07-10 11:16:42 -0700836/*
837 * Submit the queued invalidation descriptor to the remapping
838 * hardware unit and wait for its completion.
839 */
Yu Zhao704126a2009-01-04 16:28:52 +0800840int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
Suresh Siddhafe962e92008-07-10 11:16:42 -0700841{
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800842 int rc;
Suresh Siddhafe962e92008-07-10 11:16:42 -0700843 struct q_inval *qi = iommu->qi;
844 struct qi_desc *hw, wait_desc;
845 int wait_index, index;
846 unsigned long flags;
847
848 if (!qi)
Yu Zhao704126a2009-01-04 16:28:52 +0800849 return 0;
Suresh Siddhafe962e92008-07-10 11:16:42 -0700850
851 hw = qi->desc;
852
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800853restart:
854 rc = 0;
855
Suresh Siddhaf05810c2008-10-16 16:31:54 -0700856 spin_lock_irqsave(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -0700857 while (qi->free_cnt < 3) {
Suresh Siddhaf05810c2008-10-16 16:31:54 -0700858 spin_unlock_irqrestore(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -0700859 cpu_relax();
Suresh Siddhaf05810c2008-10-16 16:31:54 -0700860 spin_lock_irqsave(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -0700861 }
862
863 index = qi->free_head;
864 wait_index = (index + 1) % QI_LENGTH;
865
866 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
867
868 hw[index] = *desc;
869
Yu Zhao704126a2009-01-04 16:28:52 +0800870 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
871 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
Suresh Siddhafe962e92008-07-10 11:16:42 -0700872 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
873
874 hw[wait_index] = wait_desc;
875
876 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
877 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
878
879 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
880 qi->free_cnt -= 2;
881
Suresh Siddhafe962e92008-07-10 11:16:42 -0700882 /*
883 * update the HW tail register indicating the presence of
884 * new descriptors.
885 */
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800886 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
Suresh Siddhafe962e92008-07-10 11:16:42 -0700887
888 while (qi->desc_status[wait_index] != QI_DONE) {
Suresh Siddhaf05810c2008-10-16 16:31:54 -0700889 /*
890 * We will leave the interrupts disabled, to prevent interrupt
891 * context to queue another cmd while a cmd is already submitted
892 * and waiting for completion on this cpu. This is to avoid
893 * a deadlock where the interrupt context can wait indefinitely
894 * for free slots in the queue.
895 */
Yu Zhao704126a2009-01-04 16:28:52 +0800896 rc = qi_check_fault(iommu, index);
897 if (rc)
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800898 break;
Yu Zhao704126a2009-01-04 16:28:52 +0800899
Suresh Siddhafe962e92008-07-10 11:16:42 -0700900 spin_unlock(&qi->q_lock);
901 cpu_relax();
902 spin_lock(&qi->q_lock);
903 }
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800904
905 qi->desc_status[index] = QI_DONE;
Suresh Siddhafe962e92008-07-10 11:16:42 -0700906
907 reclaim_free_desc(qi);
Suresh Siddhaf05810c2008-10-16 16:31:54 -0700908 spin_unlock_irqrestore(&qi->q_lock, flags);
Yu Zhao704126a2009-01-04 16:28:52 +0800909
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800910 if (rc == -EAGAIN)
911 goto restart;
912
Yu Zhao704126a2009-01-04 16:28:52 +0800913 return rc;
Suresh Siddhafe962e92008-07-10 11:16:42 -0700914}
915
916/*
917 * Flush the global interrupt entry cache.
918 */
919void qi_global_iec(struct intel_iommu *iommu)
920{
921 struct qi_desc desc;
922
923 desc.low = QI_IEC_TYPE;
924 desc.high = 0;
925
Yu Zhao704126a2009-01-04 16:28:52 +0800926 /* should never fail */
Suresh Siddhafe962e92008-07-10 11:16:42 -0700927 qi_submit_sync(&desc, iommu);
928}
929
David Woodhouse4c25a2c2009-05-10 17:16:06 +0100930void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
931 u64 type)
Youquan Song3481f212008-10-16 16:31:55 -0700932{
Youquan Song3481f212008-10-16 16:31:55 -0700933 struct qi_desc desc;
934
Youquan Song3481f212008-10-16 16:31:55 -0700935 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
936 | QI_CC_GRAN(type) | QI_CC_TYPE;
937 desc.high = 0;
938
David Woodhouse4c25a2c2009-05-10 17:16:06 +0100939 qi_submit_sync(&desc, iommu);
Youquan Song3481f212008-10-16 16:31:55 -0700940}
941
David Woodhouse1f0ef2a2009-05-10 19:58:49 +0100942void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
943 unsigned int size_order, u64 type)
Youquan Song3481f212008-10-16 16:31:55 -0700944{
945 u8 dw = 0, dr = 0;
946
947 struct qi_desc desc;
948 int ih = 0;
949
Youquan Song3481f212008-10-16 16:31:55 -0700950 if (cap_write_drain(iommu->cap))
951 dw = 1;
952
953 if (cap_read_drain(iommu->cap))
954 dr = 1;
955
956 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
957 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
958 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
959 | QI_IOTLB_AM(size_order);
960
David Woodhouse1f0ef2a2009-05-10 19:58:49 +0100961 qi_submit_sync(&desc, iommu);
Youquan Song3481f212008-10-16 16:31:55 -0700962}
963
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800964void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
965 u64 addr, unsigned mask)
966{
967 struct qi_desc desc;
968
969 if (mask) {
970 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
971 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
972 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
973 } else
974 desc.high = QI_DEV_IOTLB_ADDR(addr);
975
976 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
977 qdep = 0;
978
979 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
980 QI_DIOTLB_TYPE;
981
982 qi_submit_sync(&desc, iommu);
983}
984
Suresh Siddhafe962e92008-07-10 11:16:42 -0700985/*
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700986 * Disable Queued Invalidation interface.
987 */
988void dmar_disable_qi(struct intel_iommu *iommu)
989{
990 unsigned long flags;
991 u32 sts;
992 cycles_t start_time = get_cycles();
993
994 if (!ecap_qis(iommu->ecap))
995 return;
996
997 spin_lock_irqsave(&iommu->register_lock, flags);
998
999 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
1000 if (!(sts & DMA_GSTS_QIES))
1001 goto end;
1002
1003 /*
1004 * Give a chance to HW to complete the pending invalidation requests.
1005 */
1006 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1007 readl(iommu->reg + DMAR_IQH_REG)) &&
1008 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1009 cpu_relax();
1010
1011 iommu->gcmd &= ~DMA_GCMD_QIE;
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001012 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1013
1014 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1015 !(sts & DMA_GSTS_QIES), sts);
1016end:
1017 spin_unlock_irqrestore(&iommu->register_lock, flags);
1018}
1019
1020/*
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001021 * Enable queued invalidation.
1022 */
1023static void __dmar_enable_qi(struct intel_iommu *iommu)
1024{
David Woodhousec416daa2009-05-10 20:30:58 +01001025 u32 sts;
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001026 unsigned long flags;
1027 struct q_inval *qi = iommu->qi;
1028
1029 qi->free_head = qi->free_tail = 0;
1030 qi->free_cnt = QI_LENGTH;
1031
1032 spin_lock_irqsave(&iommu->register_lock, flags);
1033
1034 /* write zero to the tail reg */
1035 writel(0, iommu->reg + DMAR_IQT_REG);
1036
1037 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1038
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001039 iommu->gcmd |= DMA_GCMD_QIE;
David Woodhousec416daa2009-05-10 20:30:58 +01001040 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001041
1042 /* Make sure hardware complete it */
1043 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1044
1045 spin_unlock_irqrestore(&iommu->register_lock, flags);
1046}
1047
1048/*
Suresh Siddhafe962e92008-07-10 11:16:42 -07001049 * Enable Queued Invalidation interface. This is a must to support
1050 * interrupt-remapping. Also used by DMA-remapping, which replaces
1051 * register based IOTLB invalidation.
1052 */
1053int dmar_enable_qi(struct intel_iommu *iommu)
1054{
Suresh Siddhafe962e92008-07-10 11:16:42 -07001055 struct q_inval *qi;
1056
1057 if (!ecap_qis(iommu->ecap))
1058 return -ENOENT;
1059
1060 /*
1061 * queued invalidation is already setup and enabled.
1062 */
1063 if (iommu->qi)
1064 return 0;
1065
Suresh Siddhafa4b57c2009-03-16 17:05:05 -07001066 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001067 if (!iommu->qi)
1068 return -ENOMEM;
1069
1070 qi = iommu->qi;
1071
Suresh Siddhafa4b57c2009-03-16 17:05:05 -07001072 qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
Suresh Siddhafe962e92008-07-10 11:16:42 -07001073 if (!qi->desc) {
1074 kfree(qi);
1075 iommu->qi = 0;
1076 return -ENOMEM;
1077 }
1078
Suresh Siddhafa4b57c2009-03-16 17:05:05 -07001079 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001080 if (!qi->desc_status) {
1081 free_page((unsigned long) qi->desc);
1082 kfree(qi);
1083 iommu->qi = 0;
1084 return -ENOMEM;
1085 }
1086
1087 qi->free_head = qi->free_tail = 0;
1088 qi->free_cnt = QI_LENGTH;
1089
1090 spin_lock_init(&qi->q_lock);
1091
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001092 __dmar_enable_qi(iommu);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001093
1094 return 0;
1095}
Suresh Siddha0ac24912009-03-16 17:04:54 -07001096
1097/* iommu interrupt handling. Most stuff are MSI-like. */
1098
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001099enum faulttype {
1100 DMA_REMAP,
1101 INTR_REMAP,
1102 UNKNOWN,
1103};
1104
1105static const char *dma_remap_fault_reasons[] =
Suresh Siddha0ac24912009-03-16 17:04:54 -07001106{
1107 "Software",
1108 "Present bit in root entry is clear",
1109 "Present bit in context entry is clear",
1110 "Invalid context entry",
1111 "Access beyond MGAW",
1112 "PTE Write access is not set",
1113 "PTE Read access is not set",
1114 "Next page table ptr is invalid",
1115 "Root table address invalid",
1116 "Context table ptr is invalid",
1117 "non-zero reserved fields in RTP",
1118 "non-zero reserved fields in CTP",
1119 "non-zero reserved fields in PTE",
1120};
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001121
1122static const char *intr_remap_fault_reasons[] =
1123{
1124 "Detected reserved fields in the decoded interrupt-remapped request",
1125 "Interrupt index exceeded the interrupt-remapping table size",
1126 "Present field in the IRTE entry is clear",
1127 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1128 "Detected reserved fields in the IRTE entry",
1129 "Blocked a compatibility format interrupt request",
1130 "Blocked an interrupt request due to source-id verification failure",
1131};
1132
Suresh Siddha0ac24912009-03-16 17:04:54 -07001133#define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1134
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001135const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001136{
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001137 if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1138 ARRAY_SIZE(intr_remap_fault_reasons))) {
1139 *fault_type = INTR_REMAP;
1140 return intr_remap_fault_reasons[fault_reason - 0x20];
1141 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1142 *fault_type = DMA_REMAP;
1143 return dma_remap_fault_reasons[fault_reason];
1144 } else {
1145 *fault_type = UNKNOWN;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001146 return "Unknown";
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001147 }
Suresh Siddha0ac24912009-03-16 17:04:54 -07001148}
1149
1150void dmar_msi_unmask(unsigned int irq)
1151{
1152 struct intel_iommu *iommu = get_irq_data(irq);
1153 unsigned long flag;
1154
1155 /* unmask it */
1156 spin_lock_irqsave(&iommu->register_lock, flag);
1157 writel(0, iommu->reg + DMAR_FECTL_REG);
1158 /* Read a reg to force flush the post write */
1159 readl(iommu->reg + DMAR_FECTL_REG);
1160 spin_unlock_irqrestore(&iommu->register_lock, flag);
1161}
1162
1163void dmar_msi_mask(unsigned int irq)
1164{
1165 unsigned long flag;
1166 struct intel_iommu *iommu = get_irq_data(irq);
1167
1168 /* mask it */
1169 spin_lock_irqsave(&iommu->register_lock, flag);
1170 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1171 /* Read a reg to force flush the post write */
1172 readl(iommu->reg + DMAR_FECTL_REG);
1173 spin_unlock_irqrestore(&iommu->register_lock, flag);
1174}
1175
1176void dmar_msi_write(int irq, struct msi_msg *msg)
1177{
1178 struct intel_iommu *iommu = get_irq_data(irq);
1179 unsigned long flag;
1180
1181 spin_lock_irqsave(&iommu->register_lock, flag);
1182 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1183 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1184 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1185 spin_unlock_irqrestore(&iommu->register_lock, flag);
1186}
1187
1188void dmar_msi_read(int irq, struct msi_msg *msg)
1189{
1190 struct intel_iommu *iommu = get_irq_data(irq);
1191 unsigned long flag;
1192
1193 spin_lock_irqsave(&iommu->register_lock, flag);
1194 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1195 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1196 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1197 spin_unlock_irqrestore(&iommu->register_lock, flag);
1198}
1199
1200static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1201 u8 fault_reason, u16 source_id, unsigned long long addr)
1202{
1203 const char *reason;
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001204 int fault_type;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001205
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001206 reason = dmar_get_fault_reason(fault_reason, &fault_type);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001207
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001208 if (fault_type == INTR_REMAP)
1209 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1210 "fault index %llx\n"
1211 "INTR-REMAP:[fault reason %02d] %s\n",
1212 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1213 PCI_FUNC(source_id & 0xFF), addr >> 48,
1214 fault_reason, reason);
1215 else
1216 printk(KERN_ERR
1217 "DMAR:[%s] Request device [%02x:%02x.%d] "
1218 "fault addr %llx \n"
1219 "DMAR:[fault reason %02d] %s\n",
1220 (type ? "DMA Read" : "DMA Write"),
1221 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1222 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001223 return 0;
1224}
1225
1226#define PRIMARY_FAULT_REG_LEN (16)
Suresh Siddha1531a6a2009-03-16 17:04:57 -07001227irqreturn_t dmar_fault(int irq, void *dev_id)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001228{
1229 struct intel_iommu *iommu = dev_id;
1230 int reg, fault_index;
1231 u32 fault_status;
1232 unsigned long flag;
1233
1234 spin_lock_irqsave(&iommu->register_lock, flag);
1235 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001236 if (fault_status)
1237 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1238 fault_status);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001239
1240 /* TBD: ignore advanced fault log currently */
1241 if (!(fault_status & DMA_FSTS_PPF))
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001242 goto clear_rest;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001243
1244 fault_index = dma_fsts_fault_record_index(fault_status);
1245 reg = cap_fault_reg_offset(iommu->cap);
1246 while (1) {
1247 u8 fault_reason;
1248 u16 source_id;
1249 u64 guest_addr;
1250 int type;
1251 u32 data;
1252
1253 /* highest 32 bits */
1254 data = readl(iommu->reg + reg +
1255 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1256 if (!(data & DMA_FRCD_F))
1257 break;
1258
1259 fault_reason = dma_frcd_fault_reason(data);
1260 type = dma_frcd_type(data);
1261
1262 data = readl(iommu->reg + reg +
1263 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1264 source_id = dma_frcd_source_id(data);
1265
1266 guest_addr = dmar_readq(iommu->reg + reg +
1267 fault_index * PRIMARY_FAULT_REG_LEN);
1268 guest_addr = dma_frcd_page_addr(guest_addr);
1269 /* clear the fault */
1270 writel(DMA_FRCD_F, iommu->reg + reg +
1271 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1272
1273 spin_unlock_irqrestore(&iommu->register_lock, flag);
1274
1275 dmar_fault_do_one(iommu, type, fault_reason,
1276 source_id, guest_addr);
1277
1278 fault_index++;
Troy Heber8211a7b2009-08-19 15:26:11 -06001279 if (fault_index >= cap_num_fault_regs(iommu->cap))
Suresh Siddha0ac24912009-03-16 17:04:54 -07001280 fault_index = 0;
1281 spin_lock_irqsave(&iommu->register_lock, flag);
1282 }
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001283clear_rest:
1284 /* clear all the other faults */
Suresh Siddha0ac24912009-03-16 17:04:54 -07001285 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001286 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001287
1288 spin_unlock_irqrestore(&iommu->register_lock, flag);
1289 return IRQ_HANDLED;
1290}
1291
1292int dmar_set_interrupt(struct intel_iommu *iommu)
1293{
1294 int irq, ret;
1295
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001296 /*
1297 * Check if the fault interrupt is already initialized.
1298 */
1299 if (iommu->irq)
1300 return 0;
1301
Suresh Siddha0ac24912009-03-16 17:04:54 -07001302 irq = create_irq();
1303 if (!irq) {
1304 printk(KERN_ERR "IOMMU: no free vectors\n");
1305 return -EINVAL;
1306 }
1307
1308 set_irq_data(irq, iommu);
1309 iommu->irq = irq;
1310
1311 ret = arch_setup_dmar_msi(irq);
1312 if (ret) {
1313 set_irq_data(irq, NULL);
1314 iommu->irq = 0;
1315 destroy_irq(irq);
Chris Wrightdd726432009-05-13 15:55:52 -07001316 return ret;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001317 }
1318
Suresh Siddha0ac24912009-03-16 17:04:54 -07001319 ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1320 if (ret)
1321 printk(KERN_ERR "IOMMU: can't request irq\n");
1322 return ret;
1323}
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001324
1325int __init enable_drhd_fault_handling(void)
1326{
1327 struct dmar_drhd_unit *drhd;
1328
1329 /*
1330 * Enable fault control interrupt.
1331 */
1332 for_each_drhd_unit(drhd) {
1333 int ret;
1334 struct intel_iommu *iommu = drhd->iommu;
1335 ret = dmar_set_interrupt(iommu);
1336
1337 if (ret) {
1338 printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1339 " interrupt, ret %d\n",
1340 (unsigned long long)drhd->reg_base_addr, ret);
1341 return -1;
1342 }
1343 }
1344
1345 return 0;
1346}
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001347
1348/*
1349 * Re-enable Queued Invalidation interface.
1350 */
1351int dmar_reenable_qi(struct intel_iommu *iommu)
1352{
1353 if (!ecap_qis(iommu->ecap))
1354 return -ENOENT;
1355
1356 if (!iommu->qi)
1357 return -ENOENT;
1358
1359 /*
1360 * First disable queued invalidation.
1361 */
1362 dmar_disable_qi(iommu);
1363 /*
1364 * Then enable queued invalidation again. Since there is no pending
1365 * invalidation requests now, it's safe to re-enable queued
1366 * invalidation.
1367 */
1368 __dmar_enable_qi(iommu);
1369
1370 return 0;
1371}
Youquan Song074835f2009-09-09 12:05:39 -04001372
1373/*
1374 * Check interrupt remapping support in DMAR table description.
1375 */
1376int dmar_ir_support(void)
1377{
1378 struct acpi_table_dmar *dmar;
1379 dmar = (struct acpi_table_dmar *)dmar_tbl;
1380 return dmar->flags & 0x1;
1381}