Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2010 |
| 3 | * by Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> |
| 4 | * |
| 5 | * This code provides a IOMMU for Xen PV guests with PCI passthrough. |
| 6 | * |
| 7 | * This program is free software; you can redistribute it and/or modify |
| 8 | * it under the terms of the GNU General Public License v2.0 as published by |
| 9 | * the Free Software Foundation |
| 10 | * |
| 11 | * This program is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | * GNU General Public License for more details. |
| 15 | * |
| 16 | * PV guests under Xen are running in an non-contiguous memory architecture. |
| 17 | * |
| 18 | * When PCI pass-through is utilized, this necessitates an IOMMU for |
| 19 | * translating bus (DMA) to virtual and vice-versa and also providing a |
| 20 | * mechanism to have contiguous pages for device drivers operations (say DMA |
| 21 | * operations). |
| 22 | * |
| 23 | * Specifically, under Xen the Linux idea of pages is an illusion. It |
| 24 | * assumes that pages start at zero and go up to the available memory. To |
| 25 | * help with that, the Linux Xen MMU provides a lookup mechanism to |
| 26 | * translate the page frame numbers (PFN) to machine frame numbers (MFN) |
| 27 | * and vice-versa. The MFN are the "real" frame numbers. Furthermore |
| 28 | * memory is not contiguous. Xen hypervisor stitches memory for guests |
| 29 | * from different pools, which means there is no guarantee that PFN==MFN |
| 30 | * and PFN+1==MFN+1. Lastly with Xen 4.0, pages (in debug mode) are |
| 31 | * allocated in descending order (high to low), meaning the guest might |
| 32 | * never get any MFN's under the 4GB mark. |
| 33 | * |
| 34 | */ |
| 35 | |
| 36 | #include <linux/bootmem.h> |
| 37 | #include <linux/dma-mapping.h> |
| 38 | #include <xen/swiotlb-xen.h> |
| 39 | #include <xen/page.h> |
| 40 | #include <xen/xen-ops.h> |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 41 | #include <xen/hvc-console.h> |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 42 | /* |
| 43 | * Used to do a quick range check in swiotlb_tbl_unmap_single and |
| 44 | * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this |
| 45 | * API. |
| 46 | */ |
| 47 | |
| 48 | static char *xen_io_tlb_start, *xen_io_tlb_end; |
| 49 | static unsigned long xen_io_tlb_nslabs; |
| 50 | /* |
| 51 | * Quick lookup value of the bus address of the IOTLB. |
| 52 | */ |
| 53 | |
| 54 | u64 start_dma_addr; |
| 55 | |
| 56 | static dma_addr_t xen_phys_to_bus(phys_addr_t paddr) |
| 57 | { |
Justin P. Mattock | 6eab04a | 2011-04-08 19:49:08 -0700 | [diff] [blame] | 58 | return phys_to_machine(XPADDR(paddr)).maddr; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 59 | } |
| 60 | |
| 61 | static phys_addr_t xen_bus_to_phys(dma_addr_t baddr) |
| 62 | { |
| 63 | return machine_to_phys(XMADDR(baddr)).paddr; |
| 64 | } |
| 65 | |
| 66 | static dma_addr_t xen_virt_to_bus(void *address) |
| 67 | { |
| 68 | return xen_phys_to_bus(virt_to_phys(address)); |
| 69 | } |
| 70 | |
| 71 | static int check_pages_physically_contiguous(unsigned long pfn, |
| 72 | unsigned int offset, |
| 73 | size_t length) |
| 74 | { |
| 75 | unsigned long next_mfn; |
| 76 | int i; |
| 77 | int nr_pages; |
| 78 | |
| 79 | next_mfn = pfn_to_mfn(pfn); |
| 80 | nr_pages = (offset + length + PAGE_SIZE-1) >> PAGE_SHIFT; |
| 81 | |
| 82 | for (i = 1; i < nr_pages; i++) { |
| 83 | if (pfn_to_mfn(++pfn) != ++next_mfn) |
| 84 | return 0; |
| 85 | } |
| 86 | return 1; |
| 87 | } |
| 88 | |
| 89 | static int range_straddles_page_boundary(phys_addr_t p, size_t size) |
| 90 | { |
| 91 | unsigned long pfn = PFN_DOWN(p); |
| 92 | unsigned int offset = p & ~PAGE_MASK; |
| 93 | |
| 94 | if (offset + size <= PAGE_SIZE) |
| 95 | return 0; |
| 96 | if (check_pages_physically_contiguous(pfn, offset, size)) |
| 97 | return 0; |
| 98 | return 1; |
| 99 | } |
| 100 | |
| 101 | static int is_xen_swiotlb_buffer(dma_addr_t dma_addr) |
| 102 | { |
| 103 | unsigned long mfn = PFN_DOWN(dma_addr); |
| 104 | unsigned long pfn = mfn_to_local_pfn(mfn); |
| 105 | phys_addr_t paddr; |
| 106 | |
| 107 | /* If the address is outside our domain, it CAN |
| 108 | * have the same virtual address as another address |
| 109 | * in our domain. Therefore _only_ check address within our domain. |
| 110 | */ |
| 111 | if (pfn_valid(pfn)) { |
| 112 | paddr = PFN_PHYS(pfn); |
| 113 | return paddr >= virt_to_phys(xen_io_tlb_start) && |
| 114 | paddr < virt_to_phys(xen_io_tlb_end); |
| 115 | } |
| 116 | return 0; |
| 117 | } |
| 118 | |
| 119 | static int max_dma_bits = 32; |
| 120 | |
| 121 | static int |
| 122 | xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs) |
| 123 | { |
| 124 | int i, rc; |
| 125 | int dma_bits; |
| 126 | |
| 127 | dma_bits = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT) + PAGE_SHIFT; |
| 128 | |
| 129 | i = 0; |
| 130 | do { |
| 131 | int slabs = min(nslabs - i, (unsigned long)IO_TLB_SEGSIZE); |
| 132 | |
| 133 | do { |
| 134 | rc = xen_create_contiguous_region( |
| 135 | (unsigned long)buf + (i << IO_TLB_SHIFT), |
| 136 | get_order(slabs << IO_TLB_SHIFT), |
| 137 | dma_bits); |
| 138 | } while (rc && dma_bits++ < max_dma_bits); |
| 139 | if (rc) |
| 140 | return rc; |
| 141 | |
| 142 | i += slabs; |
| 143 | } while (i < nslabs); |
| 144 | return 0; |
| 145 | } |
| 146 | |
| 147 | void __init xen_swiotlb_init(int verbose) |
| 148 | { |
| 149 | unsigned long bytes; |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 150 | int rc = -ENOMEM; |
FUJITA Tomonori | 5f98ecd | 2011-06-05 11:47:29 +0900 | [diff] [blame] | 151 | unsigned long nr_tbl; |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 152 | char *m = NULL; |
| 153 | unsigned int repeat = 3; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 154 | |
FUJITA Tomonori | 5f98ecd | 2011-06-05 11:47:29 +0900 | [diff] [blame] | 155 | nr_tbl = swioltb_nr_tbl(); |
| 156 | if (nr_tbl) |
| 157 | xen_io_tlb_nslabs = nr_tbl; |
| 158 | else { |
| 159 | xen_io_tlb_nslabs = (64 * 1024 * 1024 >> IO_TLB_SHIFT); |
| 160 | xen_io_tlb_nslabs = ALIGN(xen_io_tlb_nslabs, IO_TLB_SEGSIZE); |
| 161 | } |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 162 | retry: |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 163 | bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT; |
| 164 | |
| 165 | /* |
| 166 | * Get IO TLB memory from any location. |
| 167 | */ |
| 168 | xen_io_tlb_start = alloc_bootmem(bytes); |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 169 | if (!xen_io_tlb_start) { |
| 170 | m = "Cannot allocate Xen-SWIOTLB buffer!\n"; |
| 171 | goto error; |
| 172 | } |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 173 | xen_io_tlb_end = xen_io_tlb_start + bytes; |
| 174 | /* |
| 175 | * And replace that memory with pages under 4GB. |
| 176 | */ |
| 177 | rc = xen_swiotlb_fixup(xen_io_tlb_start, |
| 178 | bytes, |
| 179 | xen_io_tlb_nslabs); |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 180 | if (rc) { |
| 181 | free_bootmem(__pa(xen_io_tlb_start), bytes); |
| 182 | m = "Failed to get contiguous memory for DMA from Xen!\n"\ |
| 183 | "You either: don't have the permissions, do not have"\ |
| 184 | " enough free memory under 4GB, or the hypervisor memory"\ |
| 185 | "is too fragmented!"; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 186 | goto error; |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 187 | } |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 188 | start_dma_addr = xen_virt_to_bus(xen_io_tlb_start); |
| 189 | swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs, verbose); |
| 190 | |
| 191 | return; |
| 192 | error: |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 193 | if (repeat--) { |
| 194 | xen_io_tlb_nslabs = max(1024UL, /* Min is 2MB */ |
| 195 | (xen_io_tlb_nslabs >> 1)); |
| 196 | printk(KERN_INFO "Xen-SWIOTLB: Lowering to %luMB\n", |
| 197 | (xen_io_tlb_nslabs << IO_TLB_SHIFT) >> 20); |
| 198 | goto retry; |
| 199 | } |
Randy Dunlap | 61ca798 | 2011-08-11 13:57:07 -0700 | [diff] [blame] | 200 | xen_raw_printk("%s (rc:%d)", m, rc); |
| 201 | panic("%s (rc:%d)", m, rc); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 202 | } |
| 203 | |
| 204 | void * |
| 205 | xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size, |
| 206 | dma_addr_t *dma_handle, gfp_t flags) |
| 207 | { |
| 208 | void *ret; |
| 209 | int order = get_order(size); |
| 210 | u64 dma_mask = DMA_BIT_MASK(32); |
| 211 | unsigned long vstart; |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 212 | phys_addr_t phys; |
| 213 | dma_addr_t dev_addr; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 214 | |
| 215 | /* |
| 216 | * Ignore region specifiers - the kernel's ideas of |
| 217 | * pseudo-phys memory layout has nothing to do with the |
| 218 | * machine physical layout. We can't allocate highmem |
| 219 | * because we can't return a pointer to it. |
| 220 | */ |
| 221 | flags &= ~(__GFP_DMA | __GFP_HIGHMEM); |
| 222 | |
| 223 | if (dma_alloc_from_coherent(hwdev, size, dma_handle, &ret)) |
| 224 | return ret; |
| 225 | |
| 226 | vstart = __get_free_pages(flags, order); |
| 227 | ret = (void *)vstart; |
| 228 | |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 229 | if (!ret) |
| 230 | return ret; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 231 | |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 232 | if (hwdev && hwdev->coherent_dma_mask) |
| 233 | dma_mask = hwdev->coherent_dma_mask; |
| 234 | |
| 235 | phys = virt_to_phys(ret); |
| 236 | dev_addr = xen_phys_to_bus(phys); |
| 237 | if (((dev_addr + size - 1 <= dma_mask)) && |
| 238 | !range_straddles_page_boundary(phys, size)) |
| 239 | *dma_handle = dev_addr; |
| 240 | else { |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 241 | if (xen_create_contiguous_region(vstart, order, |
| 242 | fls64(dma_mask)) != 0) { |
| 243 | free_pages(vstart, order); |
| 244 | return NULL; |
| 245 | } |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 246 | *dma_handle = virt_to_machine(ret).maddr; |
| 247 | } |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 248 | memset(ret, 0, size); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 249 | return ret; |
| 250 | } |
| 251 | EXPORT_SYMBOL_GPL(xen_swiotlb_alloc_coherent); |
| 252 | |
| 253 | void |
| 254 | xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, |
| 255 | dma_addr_t dev_addr) |
| 256 | { |
| 257 | int order = get_order(size); |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 258 | phys_addr_t phys; |
| 259 | u64 dma_mask = DMA_BIT_MASK(32); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 260 | |
| 261 | if (dma_release_from_coherent(hwdev, order, vaddr)) |
| 262 | return; |
| 263 | |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 264 | if (hwdev && hwdev->coherent_dma_mask) |
| 265 | dma_mask = hwdev->coherent_dma_mask; |
| 266 | |
| 267 | phys = virt_to_phys(vaddr); |
| 268 | |
| 269 | if (((dev_addr + size - 1 > dma_mask)) || |
| 270 | range_straddles_page_boundary(phys, size)) |
| 271 | xen_destroy_contiguous_region((unsigned long)vaddr, order); |
| 272 | |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 273 | free_pages((unsigned long)vaddr, order); |
| 274 | } |
| 275 | EXPORT_SYMBOL_GPL(xen_swiotlb_free_coherent); |
| 276 | |
| 277 | |
| 278 | /* |
| 279 | * Map a single buffer of the indicated size for DMA in streaming mode. The |
| 280 | * physical address to use is returned. |
| 281 | * |
| 282 | * Once the device is given the dma address, the device owns this memory until |
| 283 | * either xen_swiotlb_unmap_page or xen_swiotlb_dma_sync_single is performed. |
| 284 | */ |
| 285 | dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, |
| 286 | unsigned long offset, size_t size, |
| 287 | enum dma_data_direction dir, |
| 288 | struct dma_attrs *attrs) |
| 289 | { |
| 290 | phys_addr_t phys = page_to_phys(page) + offset; |
| 291 | dma_addr_t dev_addr = xen_phys_to_bus(phys); |
| 292 | void *map; |
| 293 | |
| 294 | BUG_ON(dir == DMA_NONE); |
| 295 | /* |
| 296 | * If the address happens to be in the device's DMA window, |
| 297 | * we can safely return the device addr and not worry about bounce |
| 298 | * buffering it. |
| 299 | */ |
| 300 | if (dma_capable(dev, dev_addr, size) && |
| 301 | !range_straddles_page_boundary(phys, size) && !swiotlb_force) |
| 302 | return dev_addr; |
| 303 | |
| 304 | /* |
| 305 | * Oh well, have to allocate and map a bounce buffer. |
| 306 | */ |
| 307 | map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir); |
| 308 | if (!map) |
| 309 | return DMA_ERROR_CODE; |
| 310 | |
| 311 | dev_addr = xen_virt_to_bus(map); |
| 312 | |
| 313 | /* |
| 314 | * Ensure that the address returned is DMA'ble |
| 315 | */ |
Konrad Rzeszutek Wilk | ab2a47b | 2011-07-22 12:51:48 -0400 | [diff] [blame] | 316 | if (!dma_capable(dev, dev_addr, size)) { |
| 317 | swiotlb_tbl_unmap_single(dev, map, size, dir); |
| 318 | dev_addr = 0; |
| 319 | } |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 320 | return dev_addr; |
| 321 | } |
| 322 | EXPORT_SYMBOL_GPL(xen_swiotlb_map_page); |
| 323 | |
| 324 | /* |
| 325 | * Unmap a single streaming mode DMA translation. The dma_addr and size must |
| 326 | * match what was provided for in a previous xen_swiotlb_map_page call. All |
| 327 | * other usages are undefined. |
| 328 | * |
| 329 | * After this call, reads by the cpu to the buffer are guaranteed to see |
| 330 | * whatever the device wrote there. |
| 331 | */ |
| 332 | static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr, |
| 333 | size_t size, enum dma_data_direction dir) |
| 334 | { |
| 335 | phys_addr_t paddr = xen_bus_to_phys(dev_addr); |
| 336 | |
| 337 | BUG_ON(dir == DMA_NONE); |
| 338 | |
| 339 | /* NOTE: We use dev_addr here, not paddr! */ |
| 340 | if (is_xen_swiotlb_buffer(dev_addr)) { |
| 341 | swiotlb_tbl_unmap_single(hwdev, phys_to_virt(paddr), size, dir); |
| 342 | return; |
| 343 | } |
| 344 | |
| 345 | if (dir != DMA_FROM_DEVICE) |
| 346 | return; |
| 347 | |
| 348 | /* |
| 349 | * phys_to_virt doesn't work with hihgmem page but we could |
| 350 | * call dma_mark_clean() with hihgmem page here. However, we |
| 351 | * are fine since dma_mark_clean() is null on POWERPC. We can |
| 352 | * make dma_mark_clean() take a physical address if necessary. |
| 353 | */ |
| 354 | dma_mark_clean(phys_to_virt(paddr), size); |
| 355 | } |
| 356 | |
| 357 | void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, |
| 358 | size_t size, enum dma_data_direction dir, |
| 359 | struct dma_attrs *attrs) |
| 360 | { |
| 361 | xen_unmap_single(hwdev, dev_addr, size, dir); |
| 362 | } |
| 363 | EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_page); |
| 364 | |
| 365 | /* |
| 366 | * Make physical memory consistent for a single streaming mode DMA translation |
| 367 | * after a transfer. |
| 368 | * |
| 369 | * If you perform a xen_swiotlb_map_page() but wish to interrogate the buffer |
| 370 | * using the cpu, yet do not wish to teardown the dma mapping, you must |
| 371 | * call this function before doing so. At the next point you give the dma |
| 372 | * address back to the card, you must first perform a |
| 373 | * xen_swiotlb_dma_sync_for_device, and then the device again owns the buffer |
| 374 | */ |
| 375 | static void |
| 376 | xen_swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, |
| 377 | size_t size, enum dma_data_direction dir, |
| 378 | enum dma_sync_target target) |
| 379 | { |
| 380 | phys_addr_t paddr = xen_bus_to_phys(dev_addr); |
| 381 | |
| 382 | BUG_ON(dir == DMA_NONE); |
| 383 | |
| 384 | /* NOTE: We use dev_addr here, not paddr! */ |
| 385 | if (is_xen_swiotlb_buffer(dev_addr)) { |
| 386 | swiotlb_tbl_sync_single(hwdev, phys_to_virt(paddr), size, dir, |
| 387 | target); |
| 388 | return; |
| 389 | } |
| 390 | |
| 391 | if (dir != DMA_FROM_DEVICE) |
| 392 | return; |
| 393 | |
| 394 | dma_mark_clean(phys_to_virt(paddr), size); |
| 395 | } |
| 396 | |
| 397 | void |
| 398 | xen_swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, |
| 399 | size_t size, enum dma_data_direction dir) |
| 400 | { |
| 401 | xen_swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); |
| 402 | } |
| 403 | EXPORT_SYMBOL_GPL(xen_swiotlb_sync_single_for_cpu); |
| 404 | |
| 405 | void |
| 406 | xen_swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, |
| 407 | size_t size, enum dma_data_direction dir) |
| 408 | { |
| 409 | xen_swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); |
| 410 | } |
| 411 | EXPORT_SYMBOL_GPL(xen_swiotlb_sync_single_for_device); |
| 412 | |
| 413 | /* |
| 414 | * Map a set of buffers described by scatterlist in streaming mode for DMA. |
| 415 | * This is the scatter-gather version of the above xen_swiotlb_map_page |
| 416 | * interface. Here the scatter gather list elements are each tagged with the |
| 417 | * appropriate dma address and length. They are obtained via |
| 418 | * sg_dma_{address,length}(SG). |
| 419 | * |
| 420 | * NOTE: An implementation may be able to use a smaller number of |
| 421 | * DMA address/length pairs than there are SG table elements. |
| 422 | * (for example via virtual mapping capabilities) |
| 423 | * The routine returns the number of addr/length pairs actually |
| 424 | * used, at most nents. |
| 425 | * |
| 426 | * Device ownership issues as mentioned above for xen_swiotlb_map_page are the |
| 427 | * same here. |
| 428 | */ |
| 429 | int |
| 430 | xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, |
| 431 | int nelems, enum dma_data_direction dir, |
| 432 | struct dma_attrs *attrs) |
| 433 | { |
| 434 | struct scatterlist *sg; |
| 435 | int i; |
| 436 | |
| 437 | BUG_ON(dir == DMA_NONE); |
| 438 | |
| 439 | for_each_sg(sgl, sg, nelems, i) { |
| 440 | phys_addr_t paddr = sg_phys(sg); |
| 441 | dma_addr_t dev_addr = xen_phys_to_bus(paddr); |
| 442 | |
| 443 | if (swiotlb_force || |
| 444 | !dma_capable(hwdev, dev_addr, sg->length) || |
| 445 | range_straddles_page_boundary(paddr, sg->length)) { |
| 446 | void *map = swiotlb_tbl_map_single(hwdev, |
| 447 | start_dma_addr, |
| 448 | sg_phys(sg), |
| 449 | sg->length, dir); |
| 450 | if (!map) { |
| 451 | /* Don't panic here, we expect map_sg users |
| 452 | to do proper error handling. */ |
| 453 | xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir, |
| 454 | attrs); |
| 455 | sgl[0].dma_length = 0; |
| 456 | return DMA_ERROR_CODE; |
| 457 | } |
| 458 | sg->dma_address = xen_virt_to_bus(map); |
| 459 | } else |
| 460 | sg->dma_address = dev_addr; |
| 461 | sg->dma_length = sg->length; |
| 462 | } |
| 463 | return nelems; |
| 464 | } |
| 465 | EXPORT_SYMBOL_GPL(xen_swiotlb_map_sg_attrs); |
| 466 | |
| 467 | int |
| 468 | xen_swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, |
| 469 | enum dma_data_direction dir) |
| 470 | { |
| 471 | return xen_swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL); |
| 472 | } |
| 473 | EXPORT_SYMBOL_GPL(xen_swiotlb_map_sg); |
| 474 | |
| 475 | /* |
| 476 | * Unmap a set of streaming mode DMA translations. Again, cpu read rules |
| 477 | * concerning calls here are the same as for swiotlb_unmap_page() above. |
| 478 | */ |
| 479 | void |
| 480 | xen_swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, |
| 481 | int nelems, enum dma_data_direction dir, |
| 482 | struct dma_attrs *attrs) |
| 483 | { |
| 484 | struct scatterlist *sg; |
| 485 | int i; |
| 486 | |
| 487 | BUG_ON(dir == DMA_NONE); |
| 488 | |
| 489 | for_each_sg(sgl, sg, nelems, i) |
| 490 | xen_unmap_single(hwdev, sg->dma_address, sg->dma_length, dir); |
| 491 | |
| 492 | } |
| 493 | EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_sg_attrs); |
| 494 | |
| 495 | void |
| 496 | xen_swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, |
| 497 | enum dma_data_direction dir) |
| 498 | { |
| 499 | return xen_swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL); |
| 500 | } |
| 501 | EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_sg); |
| 502 | |
| 503 | /* |
| 504 | * Make physical memory consistent for a set of streaming mode DMA translations |
| 505 | * after a transfer. |
| 506 | * |
| 507 | * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules |
| 508 | * and usage. |
| 509 | */ |
| 510 | static void |
| 511 | xen_swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl, |
| 512 | int nelems, enum dma_data_direction dir, |
| 513 | enum dma_sync_target target) |
| 514 | { |
| 515 | struct scatterlist *sg; |
| 516 | int i; |
| 517 | |
| 518 | for_each_sg(sgl, sg, nelems, i) |
| 519 | xen_swiotlb_sync_single(hwdev, sg->dma_address, |
| 520 | sg->dma_length, dir, target); |
| 521 | } |
| 522 | |
| 523 | void |
| 524 | xen_swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, |
| 525 | int nelems, enum dma_data_direction dir) |
| 526 | { |
| 527 | xen_swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); |
| 528 | } |
| 529 | EXPORT_SYMBOL_GPL(xen_swiotlb_sync_sg_for_cpu); |
| 530 | |
| 531 | void |
| 532 | xen_swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, |
| 533 | int nelems, enum dma_data_direction dir) |
| 534 | { |
| 535 | xen_swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); |
| 536 | } |
| 537 | EXPORT_SYMBOL_GPL(xen_swiotlb_sync_sg_for_device); |
| 538 | |
| 539 | int |
| 540 | xen_swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) |
| 541 | { |
| 542 | return !dma_addr; |
| 543 | } |
| 544 | EXPORT_SYMBOL_GPL(xen_swiotlb_dma_mapping_error); |
| 545 | |
| 546 | /* |
| 547 | * Return whether the given device DMA address mask can be supported |
| 548 | * properly. For example, if your device can only drive the low 24-bits |
| 549 | * during bus mastering, then you would pass 0x00ffffff as the mask to |
| 550 | * this function. |
| 551 | */ |
| 552 | int |
| 553 | xen_swiotlb_dma_supported(struct device *hwdev, u64 mask) |
| 554 | { |
| 555 | return xen_virt_to_bus(xen_io_tlb_end - 1) <= mask; |
| 556 | } |
| 557 | EXPORT_SYMBOL_GPL(xen_swiotlb_dma_supported); |