blob: 0977317c6835c2526428f61c12fcfab976650b99 [file] [log] [blame]
Robert P. J. Day96532ba2008-02-03 15:06:26 +02001#ifndef _LINUX_DMA_MAPPING_H
2#define _LINUX_DMA_MAPPING_H
Linus Torvalds1da177e2005-04-16 15:20:36 -07003
Robin Murphy002edb62015-11-06 16:32:51 -08004#include <linux/sizes.h>
Andrew Morton842fa692011-11-02 13:39:33 -07005#include <linux/string.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07006#include <linux/device.h>
7#include <linux/err.h>
Christoph Hellwige1c7e322016-01-20 15:02:05 -08008#include <linux/dma-debug.h>
Alexey Dobriyanb7f080c2011-06-16 11:01:34 +00009#include <linux/dma-direction.h>
FUJITA Tomonorif0402a22009-01-05 23:59:01 +090010#include <linux/scatterlist.h>
Christoph Hellwige1c7e322016-01-20 15:02:05 -080011#include <linux/kmemcheck.h>
12#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -070014/**
15 * List of possible attributes associated with a DMA mapping. The semantics
16 * of each attribute should be defined in Documentation/DMA-attributes.txt.
17 *
18 * DMA_ATTR_WRITE_BARRIER: DMA to a memory region with this attribute
19 * forces all pending DMA writes to complete.
20 */
21#define DMA_ATTR_WRITE_BARRIER (1UL << 0)
22/*
23 * DMA_ATTR_WEAK_ORDERING: Specifies that reads and writes to the mapping
24 * may be weakly ordered, that is that reads and writes may pass each other.
25 */
26#define DMA_ATTR_WEAK_ORDERING (1UL << 1)
27/*
28 * DMA_ATTR_WRITE_COMBINE: Specifies that writes to the mapping may be
29 * buffered to improve performance.
30 */
31#define DMA_ATTR_WRITE_COMBINE (1UL << 2)
32/*
33 * DMA_ATTR_NON_CONSISTENT: Lets the platform to choose to return either
34 * consistent or non-consistent memory as it sees fit.
35 */
36#define DMA_ATTR_NON_CONSISTENT (1UL << 3)
37/*
38 * DMA_ATTR_NO_KERNEL_MAPPING: Lets the platform to avoid creating a kernel
39 * virtual mapping for the allocated buffer.
40 */
41#define DMA_ATTR_NO_KERNEL_MAPPING (1UL << 4)
42/*
43 * DMA_ATTR_SKIP_CPU_SYNC: Allows platform code to skip synchronization of
44 * the CPU cache for the given buffer assuming that it has been already
45 * transferred to 'device' domain.
46 */
47#define DMA_ATTR_SKIP_CPU_SYNC (1UL << 5)
48/*
49 * DMA_ATTR_FORCE_CONTIGUOUS: Forces contiguous allocation of the buffer
50 * in physical memory.
51 */
52#define DMA_ATTR_FORCE_CONTIGUOUS (1UL << 6)
53/*
54 * DMA_ATTR_ALLOC_SINGLE_PAGES: This is a hint to the DMA-mapping subsystem
55 * that it's probably not worth the time to try to allocate memory to in a way
56 * that gives better TLB efficiency.
57 */
58#define DMA_ATTR_ALLOC_SINGLE_PAGES (1UL << 7)
Mauricio Faria de Oliveiraa9a62c92016-10-11 13:54:14 -070059/*
60 * DMA_ATTR_NO_WARN: This tells the DMA-mapping subsystem to suppress
61 * allocation failure reports (similarly to __GFP_NOWARN).
62 */
63#define DMA_ATTR_NO_WARN (1UL << 8)
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -070064
Bjorn Helgaas77f2ea22014-04-30 11:20:53 -060065/*
Mitchel Humpherysb2fb3662017-01-06 18:58:11 +053066 * DMA_ATTR_PRIVILEGED: used to indicate that the buffer is fully
67 * accessible at an elevated privilege level (and ideally inaccessible or
68 * at least read-only at lesser-privileged levels).
69 */
70#define DMA_ATTR_PRIVILEGED (1UL << 9)
71
72/*
Bjorn Helgaas77f2ea22014-04-30 11:20:53 -060073 * A dma_addr_t can hold any valid DMA or bus address for the platform.
74 * It can be given to a device to use as a DMA source or target. A CPU cannot
75 * reference a dma_addr_t directly because there may be translation between
76 * its physical address space and the bus address space.
77 */
FUJITA Tomonorif0402a22009-01-05 23:59:01 +090078struct dma_map_ops {
Marek Szyprowski613c4572012-03-28 16:36:27 +020079 void* (*alloc)(struct device *dev, size_t size,
80 dma_addr_t *dma_handle, gfp_t gfp,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -070081 unsigned long attrs);
Marek Szyprowski613c4572012-03-28 16:36:27 +020082 void (*free)(struct device *dev, size_t size,
83 void *vaddr, dma_addr_t dma_handle,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -070084 unsigned long attrs);
Marek Szyprowski9adc5372011-12-21 16:55:33 +010085 int (*mmap)(struct device *, struct vm_area_struct *,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -070086 void *, dma_addr_t, size_t,
87 unsigned long attrs);
Marek Szyprowski9adc5372011-12-21 16:55:33 +010088
Marek Szyprowskid2b74282012-06-13 10:05:52 +020089 int (*get_sgtable)(struct device *dev, struct sg_table *sgt, void *,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -070090 dma_addr_t, size_t, unsigned long attrs);
Marek Szyprowskid2b74282012-06-13 10:05:52 +020091
FUJITA Tomonorif0402a22009-01-05 23:59:01 +090092 dma_addr_t (*map_page)(struct device *dev, struct page *page,
93 unsigned long offset, size_t size,
94 enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -070095 unsigned long attrs);
FUJITA Tomonorif0402a22009-01-05 23:59:01 +090096 void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
97 size_t size, enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -070098 unsigned long attrs);
Ricardo Ribalda Delgado04abab62015-02-11 13:53:15 +010099 /*
100 * map_sg returns 0 on error and a value > 0 on success.
101 * It should never return a value < 0.
102 */
FUJITA Tomonorif0402a22009-01-05 23:59:01 +0900103 int (*map_sg)(struct device *dev, struct scatterlist *sg,
104 int nents, enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700105 unsigned long attrs);
FUJITA Tomonorif0402a22009-01-05 23:59:01 +0900106 void (*unmap_sg)(struct device *dev,
107 struct scatterlist *sg, int nents,
108 enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700109 unsigned long attrs);
Niklas Söderlundba409b32016-08-10 13:22:14 +0200110 dma_addr_t (*map_resource)(struct device *dev, phys_addr_t phys_addr,
111 size_t size, enum dma_data_direction dir,
112 unsigned long attrs);
113 void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle,
114 size_t size, enum dma_data_direction dir,
115 unsigned long attrs);
FUJITA Tomonorif0402a22009-01-05 23:59:01 +0900116 void (*sync_single_for_cpu)(struct device *dev,
117 dma_addr_t dma_handle, size_t size,
118 enum dma_data_direction dir);
119 void (*sync_single_for_device)(struct device *dev,
120 dma_addr_t dma_handle, size_t size,
121 enum dma_data_direction dir);
FUJITA Tomonorif0402a22009-01-05 23:59:01 +0900122 void (*sync_sg_for_cpu)(struct device *dev,
123 struct scatterlist *sg, int nents,
124 enum dma_data_direction dir);
125 void (*sync_sg_for_device)(struct device *dev,
126 struct scatterlist *sg, int nents,
127 enum dma_data_direction dir);
128 int (*mapping_error)(struct device *dev, dma_addr_t dma_addr);
129 int (*dma_supported)(struct device *dev, u64 mask);
FUJITA Tomonorif726f30e2009-08-04 19:08:24 +0000130 int (*set_dma_mask)(struct device *dev, u64 mask);
Milton Miller3a8f7552011-06-24 09:05:23 +0000131#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
132 u64 (*get_required_mask)(struct device *dev);
133#endif
FUJITA Tomonorif0402a22009-01-05 23:59:01 +0900134 int is_phys;
135};
136
Bart Van Assche52997092017-01-20 13:04:01 -0800137extern const struct dma_map_ops dma_noop_ops;
Bart Van Assche551199a2017-01-20 13:04:07 -0800138extern const struct dma_map_ops dma_virt_ops;
Christian Borntraegera8463d42016-02-02 21:46:32 -0800139
Andrew Morton8f286c32007-10-18 03:05:07 -0700140#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
Borislav Petkov34c65382007-10-18 03:05:06 -0700141
James Bottomley32e8f702007-10-16 01:23:55 -0700142#define DMA_MASK_NONE 0x0ULL
143
Rolf Eike Beerd6bd3a32006-09-29 01:59:48 -0700144static inline int valid_dma_direction(int dma_direction)
145{
146 return ((dma_direction == DMA_BIDIRECTIONAL) ||
147 (dma_direction == DMA_TO_DEVICE) ||
148 (dma_direction == DMA_FROM_DEVICE));
149}
150
James Bottomley32e8f702007-10-16 01:23:55 -0700151static inline int is_device_dma_capable(struct device *dev)
152{
153 return dev->dma_mask != NULL && *dev->dma_mask != DMA_MASK_NONE;
154}
155
Christoph Hellwig20d666e2016-01-20 15:02:09 -0800156#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
157/*
158 * These three functions are only for dma allocator.
159 * Don't use them in device drivers.
160 */
161int dma_alloc_from_coherent(struct device *dev, ssize_t size,
162 dma_addr_t *dma_handle, void **ret);
163int dma_release_from_coherent(struct device *dev, int order, void *vaddr);
164
165int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma,
166 void *cpu_addr, size_t size, int *ret);
167#else
168#define dma_alloc_from_coherent(dev, size, handle, ret) (0)
169#define dma_release_from_coherent(dev, order, vaddr) (0)
170#define dma_mmap_from_coherent(dev, vma, vaddr, order, ret) (0)
171#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
172
Dan Williams1b0fac42007-07-15 23:40:26 -0700173#ifdef CONFIG_HAS_DMA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174#include <asm/dma-mapping.h>
Bart Van Assche815dd182017-01-20 13:04:04 -0800175static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
176{
177 if (dev && dev->dma_ops)
178 return dev->dma_ops;
179 return get_arch_dma_ops(dev ? dev->bus : NULL);
180}
181
Bart Van Asscheca6e8e12017-01-20 13:04:03 -0800182static inline void set_dma_ops(struct device *dev,
183 const struct dma_map_ops *dma_ops)
184{
185 dev->dma_ops = dma_ops;
186}
Dan Williams1b0fac42007-07-15 23:40:26 -0700187#else
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800188/*
189 * Define the dma api to allow compilation but not linking of
190 * dma dependent code. Code that depends on the dma-mapping
191 * API needs to set 'depends on HAS_DMA' in its Kconfig
192 */
Bart Van Assche52997092017-01-20 13:04:01 -0800193extern const struct dma_map_ops bad_dma_ops;
194static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800195{
196 return &bad_dma_ops;
197}
198#endif
199
200static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
201 size_t size,
202 enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700203 unsigned long attrs)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800204{
Bart Van Assche52997092017-01-20 13:04:01 -0800205 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800206 dma_addr_t addr;
207
208 kmemcheck_mark_initialized(ptr, size);
209 BUG_ON(!valid_dma_direction(dir));
210 addr = ops->map_page(dev, virt_to_page(ptr),
Geliang Tang8e994692016-01-20 15:02:12 -0800211 offset_in_page(ptr), size,
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800212 dir, attrs);
213 debug_dma_map_page(dev, virt_to_page(ptr),
Geliang Tang8e994692016-01-20 15:02:12 -0800214 offset_in_page(ptr), size,
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800215 dir, addr, true);
216 return addr;
217}
218
219static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
220 size_t size,
221 enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700222 unsigned long attrs)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800223{
Bart Van Assche52997092017-01-20 13:04:01 -0800224 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800225
226 BUG_ON(!valid_dma_direction(dir));
227 if (ops->unmap_page)
228 ops->unmap_page(dev, addr, size, dir, attrs);
229 debug_dma_unmap_page(dev, addr, size, dir, true);
230}
231
232/*
233 * dma_maps_sg_attrs returns 0 on error and > 0 on success.
234 * It should never return a value < 0.
235 */
236static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
237 int nents, enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700238 unsigned long attrs)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800239{
Bart Van Assche52997092017-01-20 13:04:01 -0800240 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800241 int i, ents;
242 struct scatterlist *s;
243
244 for_each_sg(sg, s, nents, i)
245 kmemcheck_mark_initialized(sg_virt(s), s->length);
246 BUG_ON(!valid_dma_direction(dir));
247 ents = ops->map_sg(dev, sg, nents, dir, attrs);
248 BUG_ON(ents < 0);
249 debug_dma_map_sg(dev, sg, nents, ents, dir);
250
251 return ents;
252}
253
254static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
255 int nents, enum dma_data_direction dir,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700256 unsigned long attrs)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800257{
Bart Van Assche52997092017-01-20 13:04:01 -0800258 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800259
260 BUG_ON(!valid_dma_direction(dir));
261 debug_dma_unmap_sg(dev, sg, nents, dir);
262 if (ops->unmap_sg)
263 ops->unmap_sg(dev, sg, nents, dir, attrs);
264}
265
Alexander Duyck0495c3d2016-12-14 15:05:23 -0800266static inline dma_addr_t dma_map_page_attrs(struct device *dev,
267 struct page *page,
268 size_t offset, size_t size,
269 enum dma_data_direction dir,
270 unsigned long attrs)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800271{
Bart Van Assche52997092017-01-20 13:04:01 -0800272 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800273 dma_addr_t addr;
274
275 kmemcheck_mark_initialized(page_address(page) + offset, size);
276 BUG_ON(!valid_dma_direction(dir));
Alexander Duyck0495c3d2016-12-14 15:05:23 -0800277 addr = ops->map_page(dev, page, offset, size, dir, attrs);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800278 debug_dma_map_page(dev, page, offset, size, dir, addr, false);
279
280 return addr;
281}
282
Alexander Duyck0495c3d2016-12-14 15:05:23 -0800283static inline void dma_unmap_page_attrs(struct device *dev,
284 dma_addr_t addr, size_t size,
285 enum dma_data_direction dir,
286 unsigned long attrs)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800287{
Bart Van Assche52997092017-01-20 13:04:01 -0800288 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800289
290 BUG_ON(!valid_dma_direction(dir));
291 if (ops->unmap_page)
Alexander Duyck0495c3d2016-12-14 15:05:23 -0800292 ops->unmap_page(dev, addr, size, dir, attrs);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800293 debug_dma_unmap_page(dev, addr, size, dir, false);
294}
295
Niklas Söderlund6f3d8792016-08-10 13:22:16 +0200296static inline dma_addr_t dma_map_resource(struct device *dev,
297 phys_addr_t phys_addr,
298 size_t size,
299 enum dma_data_direction dir,
300 unsigned long attrs)
301{
Bart Van Assche52997092017-01-20 13:04:01 -0800302 const struct dma_map_ops *ops = get_dma_ops(dev);
Niklas Söderlund6f3d8792016-08-10 13:22:16 +0200303 dma_addr_t addr;
304
305 BUG_ON(!valid_dma_direction(dir));
306
307 /* Don't allow RAM to be mapped */
Niklas Söderlund3757dc42016-09-29 12:02:40 +0200308 BUG_ON(pfn_valid(PHYS_PFN(phys_addr)));
Niklas Söderlund6f3d8792016-08-10 13:22:16 +0200309
310 addr = phys_addr;
311 if (ops->map_resource)
312 addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
313
314 debug_dma_map_resource(dev, phys_addr, size, dir, addr);
315
316 return addr;
317}
318
319static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
320 size_t size, enum dma_data_direction dir,
321 unsigned long attrs)
322{
Bart Van Assche52997092017-01-20 13:04:01 -0800323 const struct dma_map_ops *ops = get_dma_ops(dev);
Niklas Söderlund6f3d8792016-08-10 13:22:16 +0200324
325 BUG_ON(!valid_dma_direction(dir));
326 if (ops->unmap_resource)
327 ops->unmap_resource(dev, addr, size, dir, attrs);
328 debug_dma_unmap_resource(dev, addr, size, dir);
329}
330
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800331static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
332 size_t size,
333 enum dma_data_direction dir)
334{
Bart Van Assche52997092017-01-20 13:04:01 -0800335 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800336
337 BUG_ON(!valid_dma_direction(dir));
338 if (ops->sync_single_for_cpu)
339 ops->sync_single_for_cpu(dev, addr, size, dir);
340 debug_dma_sync_single_for_cpu(dev, addr, size, dir);
341}
342
343static inline void dma_sync_single_for_device(struct device *dev,
344 dma_addr_t addr, size_t size,
345 enum dma_data_direction dir)
346{
Bart Van Assche52997092017-01-20 13:04:01 -0800347 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800348
349 BUG_ON(!valid_dma_direction(dir));
350 if (ops->sync_single_for_device)
351 ops->sync_single_for_device(dev, addr, size, dir);
352 debug_dma_sync_single_for_device(dev, addr, size, dir);
353}
354
355static inline void dma_sync_single_range_for_cpu(struct device *dev,
356 dma_addr_t addr,
357 unsigned long offset,
358 size_t size,
359 enum dma_data_direction dir)
360{
361 const struct dma_map_ops *ops = get_dma_ops(dev);
362
363 BUG_ON(!valid_dma_direction(dir));
364 if (ops->sync_single_for_cpu)
365 ops->sync_single_for_cpu(dev, addr + offset, size, dir);
366 debug_dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
367}
368
369static inline void dma_sync_single_range_for_device(struct device *dev,
370 dma_addr_t addr,
371 unsigned long offset,
372 size_t size,
373 enum dma_data_direction dir)
374{
375 const struct dma_map_ops *ops = get_dma_ops(dev);
376
377 BUG_ON(!valid_dma_direction(dir));
378 if (ops->sync_single_for_device)
379 ops->sync_single_for_device(dev, addr + offset, size, dir);
380 debug_dma_sync_single_range_for_device(dev, addr, offset, size, dir);
381}
382
383static inline void
384dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
385 int nelems, enum dma_data_direction dir)
386{
Bart Van Assche52997092017-01-20 13:04:01 -0800387 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800388
389 BUG_ON(!valid_dma_direction(dir));
390 if (ops->sync_sg_for_cpu)
391 ops->sync_sg_for_cpu(dev, sg, nelems, dir);
392 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
393}
394
395static inline void
396dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
397 int nelems, enum dma_data_direction dir)
398{
Bart Van Assche52997092017-01-20 13:04:01 -0800399 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800400
401 BUG_ON(!valid_dma_direction(dir));
402 if (ops->sync_sg_for_device)
403 ops->sync_sg_for_device(dev, sg, nelems, dir);
404 debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
405
406}
407
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700408#define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, 0)
409#define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0)
410#define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0)
411#define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, 0)
Alexander Duyck0495c3d2016-12-14 15:05:23 -0800412#define dma_map_page(d, p, o, s, r) dma_map_page_attrs(d, p, o, s, r, 0)
413#define dma_unmap_page(d, a, s, r) dma_unmap_page_attrs(d, a, s, r, 0)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800414
415extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
416 void *cpu_addr, dma_addr_t dma_addr, size_t size);
417
418void *dma_common_contiguous_remap(struct page *page, size_t size,
419 unsigned long vm_flags,
420 pgprot_t prot, const void *caller);
421
422void *dma_common_pages_remap(struct page **pages, size_t size,
423 unsigned long vm_flags, pgprot_t prot,
424 const void *caller);
425void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags);
426
427/**
428 * dma_mmap_attrs - map a coherent DMA allocation into user space
429 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
430 * @vma: vm_area_struct describing requested user mapping
431 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
432 * @handle: device-view address returned from dma_alloc_attrs
433 * @size: size of memory originally requested in dma_alloc_attrs
434 * @attrs: attributes of mapping properties requested in dma_alloc_attrs
435 *
436 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs
437 * into user space. The coherent DMA buffer must not be freed by the
438 * driver until the user space mapping has been released.
439 */
440static inline int
441dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700442 dma_addr_t dma_addr, size_t size, unsigned long attrs)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800443{
Bart Van Assche52997092017-01-20 13:04:01 -0800444 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800445 BUG_ON(!ops);
446 if (ops->mmap)
447 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
448 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
449}
450
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700451#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800452
453int
454dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
455 void *cpu_addr, dma_addr_t dma_addr, size_t size);
456
457static inline int
458dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700459 dma_addr_t dma_addr, size_t size,
460 unsigned long attrs)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800461{
Bart Van Assche52997092017-01-20 13:04:01 -0800462 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800463 BUG_ON(!ops);
464 if (ops->get_sgtable)
465 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
466 attrs);
467 return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size);
468}
469
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700470#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800471
472#ifndef arch_dma_alloc_attrs
473#define arch_dma_alloc_attrs(dev, flag) (true)
474#endif
475
476static inline void *dma_alloc_attrs(struct device *dev, size_t size,
477 dma_addr_t *dma_handle, gfp_t flag,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700478 unsigned long attrs)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800479{
Bart Van Assche52997092017-01-20 13:04:01 -0800480 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800481 void *cpu_addr;
482
483 BUG_ON(!ops);
484
485 if (dma_alloc_from_coherent(dev, size, dma_handle, &cpu_addr))
486 return cpu_addr;
487
488 if (!arch_dma_alloc_attrs(&dev, &flag))
489 return NULL;
490 if (!ops->alloc)
491 return NULL;
492
493 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
494 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
495 return cpu_addr;
496}
497
498static inline void dma_free_attrs(struct device *dev, size_t size,
499 void *cpu_addr, dma_addr_t dma_handle,
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700500 unsigned long attrs)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800501{
Bart Van Assche52997092017-01-20 13:04:01 -0800502 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800503
504 BUG_ON(!ops);
505 WARN_ON(irqs_disabled());
506
507 if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
508 return;
509
Zhen Leid6b7eae2016-03-09 14:08:38 -0800510 if (!ops->free || !cpu_addr)
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800511 return;
512
513 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
514 ops->free(dev, size, cpu_addr, dma_handle, attrs);
515}
516
517static inline void *dma_alloc_coherent(struct device *dev, size_t size,
518 dma_addr_t *dma_handle, gfp_t flag)
519{
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700520 return dma_alloc_attrs(dev, size, dma_handle, flag, 0);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800521}
522
523static inline void dma_free_coherent(struct device *dev, size_t size,
524 void *cpu_addr, dma_addr_t dma_handle)
525{
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700526 return dma_free_attrs(dev, size, cpu_addr, dma_handle, 0);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800527}
528
529static inline void *dma_alloc_noncoherent(struct device *dev, size_t size,
530 dma_addr_t *dma_handle, gfp_t gfp)
531{
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700532 return dma_alloc_attrs(dev, size, dma_handle, gfp,
533 DMA_ATTR_NON_CONSISTENT);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800534}
535
536static inline void dma_free_noncoherent(struct device *dev, size_t size,
537 void *cpu_addr, dma_addr_t dma_handle)
538{
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700539 dma_free_attrs(dev, size, cpu_addr, dma_handle,
540 DMA_ATTR_NON_CONSISTENT);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800541}
542
543static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
544{
545 debug_dma_mapping_error(dev, dma_addr);
546
547 if (get_dma_ops(dev)->mapping_error)
548 return get_dma_ops(dev)->mapping_error(dev, dma_addr);
549
550#ifdef DMA_ERROR_CODE
551 return dma_addr == DMA_ERROR_CODE;
552#else
553 return 0;
554#endif
555}
556
557#ifndef HAVE_ARCH_DMA_SUPPORTED
558static inline int dma_supported(struct device *dev, u64 mask)
559{
Bart Van Assche52997092017-01-20 13:04:01 -0800560 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800561
562 if (!ops)
563 return 0;
564 if (!ops->dma_supported)
565 return 1;
566 return ops->dma_supported(dev, mask);
567}
568#endif
569
570#ifndef HAVE_ARCH_DMA_SET_MASK
571static inline int dma_set_mask(struct device *dev, u64 mask)
572{
Bart Van Assche52997092017-01-20 13:04:01 -0800573 const struct dma_map_ops *ops = get_dma_ops(dev);
Christoph Hellwige1c7e322016-01-20 15:02:05 -0800574
575 if (ops->set_dma_mask)
576 return ops->set_dma_mask(dev, mask);
577
578 if (!dev->dma_mask || !dma_supported(dev, mask))
579 return -EIO;
580 *dev->dma_mask = mask;
581 return 0;
582}
Dan Williams1b0fac42007-07-15 23:40:26 -0700583#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584
FUJITA Tomonori589fc9a2008-09-12 19:42:34 +0900585static inline u64 dma_get_mask(struct device *dev)
586{
FUJITA Tomonori07a2c012008-09-19 02:02:05 +0900587 if (dev && dev->dma_mask && *dev->dma_mask)
FUJITA Tomonori589fc9a2008-09-12 19:42:34 +0900588 return *dev->dma_mask;
Yang Hongyang284901a2009-04-06 19:01:15 -0700589 return DMA_BIT_MASK(32);
FUJITA Tomonori589fc9a2008-09-12 19:42:34 +0900590}
591
Rob Herring58af4a22012-03-20 14:33:01 -0500592#ifdef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK
FUJITA Tomonori710224f2010-09-22 13:04:55 -0700593int dma_set_coherent_mask(struct device *dev, u64 mask);
594#else
FUJITA Tomonori6a1961f2010-03-10 15:23:39 -0800595static inline int dma_set_coherent_mask(struct device *dev, u64 mask)
596{
597 if (!dma_supported(dev, mask))
598 return -EIO;
599 dev->coherent_dma_mask = mask;
600 return 0;
601}
FUJITA Tomonori710224f2010-09-22 13:04:55 -0700602#endif
FUJITA Tomonori6a1961f2010-03-10 15:23:39 -0800603
Russell King4aa806b2013-06-26 13:49:44 +0100604/*
605 * Set both the DMA mask and the coherent DMA mask to the same thing.
606 * Note that we don't check the return value from dma_set_coherent_mask()
607 * as the DMA API guarantees that the coherent DMA mask can be set to
608 * the same or smaller than the streaming DMA mask.
609 */
610static inline int dma_set_mask_and_coherent(struct device *dev, u64 mask)
611{
612 int rc = dma_set_mask(dev, mask);
613 if (rc == 0)
614 dma_set_coherent_mask(dev, mask);
615 return rc;
616}
617
Russell Kingfa6a8d62013-06-27 12:21:45 +0100618/*
619 * Similar to the above, except it deals with the case where the device
620 * does not have dev->dma_mask appropriately setup.
621 */
622static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
623{
624 dev->dma_mask = &dev->coherent_dma_mask;
625 return dma_set_mask_and_coherent(dev, mask);
626}
627
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628extern u64 dma_get_required_mask(struct device *dev);
629
Will Deacona3a60f82014-08-27 15:49:10 +0100630#ifndef arch_setup_dma_ops
Will Deacon97890ba2014-08-27 16:24:20 +0100631static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
Robin Murphy53c92d72016-04-07 18:42:05 +0100632 u64 size, const struct iommu_ops *iommu,
Will Deacon97890ba2014-08-27 16:24:20 +0100633 bool coherent) { }
634#endif
635
636#ifndef arch_teardown_dma_ops
637static inline void arch_teardown_dma_ops(struct device *dev) { }
Santosh Shilimkar591c1ee2014-04-24 11:30:04 -0400638#endif
639
FUJITA Tomonori6b7b6512008-02-04 22:27:55 -0800640static inline unsigned int dma_get_max_seg_size(struct device *dev)
641{
Robin Murphy002edb62015-11-06 16:32:51 -0800642 if (dev->dma_parms && dev->dma_parms->max_segment_size)
643 return dev->dma_parms->max_segment_size;
644 return SZ_64K;
FUJITA Tomonori6b7b6512008-02-04 22:27:55 -0800645}
646
647static inline unsigned int dma_set_max_seg_size(struct device *dev,
648 unsigned int size)
649{
650 if (dev->dma_parms) {
651 dev->dma_parms->max_segment_size = size;
652 return 0;
Robin Murphy002edb62015-11-06 16:32:51 -0800653 }
654 return -EIO;
FUJITA Tomonori6b7b6512008-02-04 22:27:55 -0800655}
656
FUJITA Tomonorid22a6962008-02-04 22:28:13 -0800657static inline unsigned long dma_get_seg_boundary(struct device *dev)
658{
Robin Murphy002edb62015-11-06 16:32:51 -0800659 if (dev->dma_parms && dev->dma_parms->segment_boundary_mask)
660 return dev->dma_parms->segment_boundary_mask;
661 return DMA_BIT_MASK(32);
FUJITA Tomonorid22a6962008-02-04 22:28:13 -0800662}
663
664static inline int dma_set_seg_boundary(struct device *dev, unsigned long mask)
665{
666 if (dev->dma_parms) {
667 dev->dma_parms->segment_boundary_mask = mask;
668 return 0;
Robin Murphy002edb62015-11-06 16:32:51 -0800669 }
670 return -EIO;
FUJITA Tomonorid22a6962008-02-04 22:28:13 -0800671}
672
Santosh Shilimkar00c8f162013-07-29 14:18:48 +0100673#ifndef dma_max_pfn
674static inline unsigned long dma_max_pfn(struct device *dev)
675{
676 return *dev->dma_mask >> PAGE_SHIFT;
677}
678#endif
679
Andrew Morton842fa692011-11-02 13:39:33 -0700680static inline void *dma_zalloc_coherent(struct device *dev, size_t size,
681 dma_addr_t *dma_handle, gfp_t flag)
682{
Joe Perchesede23fa2013-08-26 22:45:23 -0700683 void *ret = dma_alloc_coherent(dev, size, dma_handle,
684 flag | __GFP_ZERO);
Andrew Morton842fa692011-11-02 13:39:33 -0700685 return ret;
686}
687
Heiko Carstense259f192010-08-13 09:39:18 +0200688#ifdef CONFIG_HAS_DMA
FUJITA Tomonori4565f012010-08-10 18:03:22 -0700689static inline int dma_get_cache_alignment(void)
690{
691#ifdef ARCH_DMA_MINALIGN
692 return ARCH_DMA_MINALIGN;
693#endif
694 return 1;
695}
Heiko Carstense259f192010-08-13 09:39:18 +0200696#endif
FUJITA Tomonori4565f012010-08-10 18:03:22 -0700697
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698/* flags for the coherent memory api */
699#define DMA_MEMORY_MAP 0x01
700#define DMA_MEMORY_IO 0x02
701#define DMA_MEMORY_INCLUDES_CHILDREN 0x04
702#define DMA_MEMORY_EXCLUSIVE 0x08
703
Christoph Hellwig20d666e2016-01-20 15:02:09 -0800704#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
705int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
706 dma_addr_t device_addr, size_t size, int flags);
707void dma_release_declared_memory(struct device *dev);
708void *dma_mark_declared_memory_occupied(struct device *dev,
709 dma_addr_t device_addr, size_t size);
710#else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711static inline int
Bjorn Helgaas88a984b2014-05-20 16:54:22 -0600712dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713 dma_addr_t device_addr, size_t size, int flags)
714{
715 return 0;
716}
717
718static inline void
719dma_release_declared_memory(struct device *dev)
720{
721}
722
723static inline void *
724dma_mark_declared_memory_occupied(struct device *dev,
725 dma_addr_t device_addr, size_t size)
726{
727 return ERR_PTR(-EBUSY);
728}
Christoph Hellwig20d666e2016-01-20 15:02:09 -0800729#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730
Tejun Heo9ac78492007-01-20 16:00:26 +0900731/*
732 * Managed DMA API
733 */
734extern void *dmam_alloc_coherent(struct device *dev, size_t size,
735 dma_addr_t *dma_handle, gfp_t gfp);
736extern void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
737 dma_addr_t dma_handle);
738extern void *dmam_alloc_noncoherent(struct device *dev, size_t size,
739 dma_addr_t *dma_handle, gfp_t gfp);
740extern void dmam_free_noncoherent(struct device *dev, size_t size, void *vaddr,
741 dma_addr_t dma_handle);
Christoph Hellwig20d666e2016-01-20 15:02:09 -0800742#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
Bjorn Helgaas88a984b2014-05-20 16:54:22 -0600743extern int dmam_declare_coherent_memory(struct device *dev,
744 phys_addr_t phys_addr,
Tejun Heo9ac78492007-01-20 16:00:26 +0900745 dma_addr_t device_addr, size_t size,
746 int flags);
747extern void dmam_release_declared_memory(struct device *dev);
Christoph Hellwig20d666e2016-01-20 15:02:09 -0800748#else /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
Tejun Heo9ac78492007-01-20 16:00:26 +0900749static inline int dmam_declare_coherent_memory(struct device *dev,
Bjorn Helgaas88a984b2014-05-20 16:54:22 -0600750 phys_addr_t phys_addr, dma_addr_t device_addr,
Tejun Heo9ac78492007-01-20 16:00:26 +0900751 size_t size, gfp_t gfp)
752{
753 return 0;
754}
755
756static inline void dmam_release_declared_memory(struct device *dev)
757{
758}
Christoph Hellwig20d666e2016-01-20 15:02:09 -0800759#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
Tejun Heo9ac78492007-01-20 16:00:26 +0900760
Luis R. Rodriguezf6e45662016-01-22 18:34:22 -0800761static inline void *dma_alloc_wc(struct device *dev, size_t size,
762 dma_addr_t *dma_addr, gfp_t gfp)
Thierry Redingb4bbb102014-06-27 11:56:58 +0200763{
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700764 return dma_alloc_attrs(dev, size, dma_addr, gfp,
765 DMA_ATTR_WRITE_COMBINE);
Thierry Redingb4bbb102014-06-27 11:56:58 +0200766}
Luis R. Rodriguezf6e45662016-01-22 18:34:22 -0800767#ifndef dma_alloc_writecombine
768#define dma_alloc_writecombine dma_alloc_wc
769#endif
Thierry Redingb4bbb102014-06-27 11:56:58 +0200770
Luis R. Rodriguezf6e45662016-01-22 18:34:22 -0800771static inline void dma_free_wc(struct device *dev, size_t size,
772 void *cpu_addr, dma_addr_t dma_addr)
Thierry Redingb4bbb102014-06-27 11:56:58 +0200773{
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700774 return dma_free_attrs(dev, size, cpu_addr, dma_addr,
775 DMA_ATTR_WRITE_COMBINE);
Thierry Redingb4bbb102014-06-27 11:56:58 +0200776}
Luis R. Rodriguezf6e45662016-01-22 18:34:22 -0800777#ifndef dma_free_writecombine
778#define dma_free_writecombine dma_free_wc
779#endif
Thierry Redingb4bbb102014-06-27 11:56:58 +0200780
Luis R. Rodriguezf6e45662016-01-22 18:34:22 -0800781static inline int dma_mmap_wc(struct device *dev,
782 struct vm_area_struct *vma,
783 void *cpu_addr, dma_addr_t dma_addr,
784 size_t size)
Thierry Redingb4bbb102014-06-27 11:56:58 +0200785{
Krzysztof Kozlowski00085f12016-08-03 13:46:00 -0700786 return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size,
787 DMA_ATTR_WRITE_COMBINE);
Thierry Redingb4bbb102014-06-27 11:56:58 +0200788}
Luis R. Rodriguezf6e45662016-01-22 18:34:22 -0800789#ifndef dma_mmap_writecombine
790#define dma_mmap_writecombine dma_mmap_wc
791#endif
Arthur Kepner74bc7ce2008-04-29 01:00:30 -0700792
Andrey Smirnov24813662016-09-28 15:22:33 -0700793#if defined(CONFIG_NEED_DMA_MAP_STATE) || defined(CONFIG_DMA_API_DEBUG)
FUJITA Tomonori0acedc12010-03-10 15:23:31 -0800794#define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME) dma_addr_t ADDR_NAME
795#define DEFINE_DMA_UNMAP_LEN(LEN_NAME) __u32 LEN_NAME
796#define dma_unmap_addr(PTR, ADDR_NAME) ((PTR)->ADDR_NAME)
797#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL) (((PTR)->ADDR_NAME) = (VAL))
798#define dma_unmap_len(PTR, LEN_NAME) ((PTR)->LEN_NAME)
799#define dma_unmap_len_set(PTR, LEN_NAME, VAL) (((PTR)->LEN_NAME) = (VAL))
800#else
801#define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)
802#define DEFINE_DMA_UNMAP_LEN(LEN_NAME)
803#define dma_unmap_addr(PTR, ADDR_NAME) (0)
804#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL) do { } while (0)
805#define dma_unmap_len(PTR, LEN_NAME) (0)
806#define dma_unmap_len_set(PTR, LEN_NAME, VAL) do { } while (0)
807#endif
808
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809#endif