David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 1 | /* pci_sun4v.c: SUN4V specific PCI controller support. |
| 2 | * |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 3 | * Copyright (C) 2006, 2007, 2008 David S. Miller (davem@davemloft.net) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 4 | */ |
| 5 | |
| 6 | #include <linux/kernel.h> |
| 7 | #include <linux/types.h> |
| 8 | #include <linux/pci.h> |
| 9 | #include <linux/init.h> |
| 10 | #include <linux/slab.h> |
| 11 | #include <linux/interrupt.h> |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 12 | #include <linux/percpu.h> |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 13 | #include <linux/irq.h> |
| 14 | #include <linux/msi.h> |
Paul Gortmaker | 7b64db6 | 2011-07-18 15:57:46 -0400 | [diff] [blame] | 15 | #include <linux/export.h> |
David S. Miller | 59db810 | 2007-05-23 18:00:46 -0700 | [diff] [blame] | 16 | #include <linux/log2.h> |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 17 | #include <linux/of_device.h> |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 18 | |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 19 | #include <asm/iommu.h> |
| 20 | #include <asm/irq.h> |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 21 | #include <asm/hypervisor.h> |
David S. Miller | e87dc35 | 2006-06-21 18:18:47 -0700 | [diff] [blame] | 22 | #include <asm/prom.h> |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 23 | |
| 24 | #include "pci_impl.h" |
| 25 | #include "iommu_common.h" |
| 26 | |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 27 | #include "pci_sun4v.h" |
| 28 | |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 29 | #define DRIVER_NAME "pci_sun4v" |
| 30 | #define PFX DRIVER_NAME ": " |
| 31 | |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 32 | static unsigned long vpci_major = 1; |
| 33 | static unsigned long vpci_minor = 1; |
| 34 | |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 35 | #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 36 | |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 37 | struct iommu_batch { |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 38 | struct device *dev; /* Device mapping is for. */ |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 39 | unsigned long prot; /* IOMMU page protections */ |
| 40 | unsigned long entry; /* Index into IOTSB. */ |
| 41 | u64 *pglist; /* List of physical pages */ |
| 42 | unsigned long npages; /* Number of pages in list. */ |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 43 | }; |
| 44 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 45 | static DEFINE_PER_CPU(struct iommu_batch, iommu_batch); |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 46 | static int iommu_batch_initialized; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 47 | |
| 48 | /* Interrupts must be disabled. */ |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 49 | static inline void iommu_batch_start(struct device *dev, unsigned long prot, unsigned long entry) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 50 | { |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 51 | struct iommu_batch *p = &__get_cpu_var(iommu_batch); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 52 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 53 | p->dev = dev; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 54 | p->prot = prot; |
| 55 | p->entry = entry; |
| 56 | p->npages = 0; |
| 57 | } |
| 58 | |
| 59 | /* Interrupts must be disabled. */ |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 60 | static long iommu_batch_flush(struct iommu_batch *p) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 61 | { |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 62 | struct pci_pbm_info *pbm = p->dev->archdata.host_controller; |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 63 | unsigned long devhandle = pbm->devhandle; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 64 | unsigned long prot = p->prot; |
| 65 | unsigned long entry = p->entry; |
| 66 | u64 *pglist = p->pglist; |
| 67 | unsigned long npages = p->npages; |
| 68 | |
David S. Miller | d82965c | 2006-02-20 01:42:51 -0800 | [diff] [blame] | 69 | while (npages != 0) { |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 70 | long num; |
| 71 | |
| 72 | num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry), |
| 73 | npages, prot, __pa(pglist)); |
| 74 | if (unlikely(num < 0)) { |
| 75 | if (printk_ratelimit()) |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 76 | printk("iommu_batch_flush: IOMMU map of " |
Sam Ravnborg | 9018113 | 2009-01-06 13:19:28 -0800 | [diff] [blame] | 77 | "[%08lx:%08llx:%lx:%lx:%lx] failed with " |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 78 | "status %ld\n", |
| 79 | devhandle, HV_PCI_TSBID(0, entry), |
| 80 | npages, prot, __pa(pglist), num); |
| 81 | return -1; |
| 82 | } |
| 83 | |
| 84 | entry += num; |
| 85 | npages -= num; |
| 86 | pglist += num; |
David S. Miller | d82965c | 2006-02-20 01:42:51 -0800 | [diff] [blame] | 87 | } |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 88 | |
| 89 | p->entry = entry; |
| 90 | p->npages = 0; |
| 91 | |
| 92 | return 0; |
| 93 | } |
| 94 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 95 | static inline void iommu_batch_new_entry(unsigned long entry) |
| 96 | { |
| 97 | struct iommu_batch *p = &__get_cpu_var(iommu_batch); |
| 98 | |
| 99 | if (p->entry + p->npages == entry) |
| 100 | return; |
| 101 | if (p->entry != ~0UL) |
| 102 | iommu_batch_flush(p); |
| 103 | p->entry = entry; |
| 104 | } |
| 105 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 106 | /* Interrupts must be disabled. */ |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 107 | static inline long iommu_batch_add(u64 phys_page) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 108 | { |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 109 | struct iommu_batch *p = &__get_cpu_var(iommu_batch); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 110 | |
| 111 | BUG_ON(p->npages >= PGLIST_NENTS); |
| 112 | |
| 113 | p->pglist[p->npages++] = phys_page; |
| 114 | if (p->npages == PGLIST_NENTS) |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 115 | return iommu_batch_flush(p); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 116 | |
| 117 | return 0; |
| 118 | } |
| 119 | |
| 120 | /* Interrupts must be disabled. */ |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 121 | static inline long iommu_batch_end(void) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 122 | { |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 123 | struct iommu_batch *p = &__get_cpu_var(iommu_batch); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 124 | |
| 125 | BUG_ON(p->npages >= PGLIST_NENTS); |
| 126 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 127 | return iommu_batch_flush(p); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 128 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 129 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 130 | static void *dma_4v_alloc_coherent(struct device *dev, size_t size, |
Andrzej Pietrasiewicz | c416258 | 2012-03-27 14:56:55 +0200 | [diff] [blame] | 131 | dma_addr_t *dma_addrp, gfp_t gfp, |
| 132 | struct dma_attrs *attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 133 | { |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 134 | unsigned long flags, order, first_page, npages, n; |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 135 | struct iommu *iommu; |
| 136 | struct page *page; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 137 | void *ret; |
| 138 | long entry; |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 139 | int nid; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 140 | |
| 141 | size = IO_PAGE_ALIGN(size); |
| 142 | order = get_order(size); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 143 | if (unlikely(order >= MAX_ORDER)) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 144 | return NULL; |
| 145 | |
| 146 | npages = size >> IO_PAGE_SHIFT; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 147 | |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 148 | nid = dev->archdata.numa_node; |
| 149 | page = alloc_pages_node(nid, gfp, order); |
| 150 | if (unlikely(!page)) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 151 | return NULL; |
David S. Miller | e7a0453 | 2006-02-15 22:25:27 -0800 | [diff] [blame] | 152 | |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 153 | first_page = (unsigned long) page_address(page); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 154 | memset((char *)first_page, 0, PAGE_SIZE << order); |
| 155 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 156 | iommu = dev->archdata.iommu; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 157 | |
| 158 | spin_lock_irqsave(&iommu->lock, flags); |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 159 | entry = iommu_range_alloc(dev, iommu, npages, NULL); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 160 | spin_unlock_irqrestore(&iommu->lock, flags); |
| 161 | |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 162 | if (unlikely(entry == DMA_ERROR_CODE)) |
| 163 | goto range_alloc_fail; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 164 | |
| 165 | *dma_addrp = (iommu->page_table_map_base + |
| 166 | (entry << IO_PAGE_SHIFT)); |
| 167 | ret = (void *) first_page; |
| 168 | first_page = __pa(first_page); |
| 169 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 170 | local_irq_save(flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 171 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 172 | iommu_batch_start(dev, |
| 173 | (HV_PCI_MAP_ATTR_READ | |
| 174 | HV_PCI_MAP_ATTR_WRITE), |
| 175 | entry); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 176 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 177 | for (n = 0; n < npages; n++) { |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 178 | long err = iommu_batch_add(first_page + (n * PAGE_SIZE)); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 179 | if (unlikely(err < 0L)) |
| 180 | goto iommu_map_fail; |
| 181 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 182 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 183 | if (unlikely(iommu_batch_end() < 0L)) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 184 | goto iommu_map_fail; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 185 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 186 | local_irq_restore(flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 187 | |
| 188 | return ret; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 189 | |
| 190 | iommu_map_fail: |
| 191 | /* Interrupts are disabled. */ |
| 192 | spin_lock(&iommu->lock); |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 193 | iommu_range_free(iommu, *dma_addrp, npages); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 194 | spin_unlock_irqrestore(&iommu->lock, flags); |
| 195 | |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 196 | range_alloc_fail: |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 197 | free_pages(first_page, order); |
| 198 | return NULL; |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 199 | } |
| 200 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 201 | static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, |
Andrzej Pietrasiewicz | c416258 | 2012-03-27 14:56:55 +0200 | [diff] [blame] | 202 | dma_addr_t dvma, struct dma_attrs *attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 203 | { |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 204 | struct pci_pbm_info *pbm; |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 205 | struct iommu *iommu; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 206 | unsigned long flags, order, npages, entry; |
| 207 | u32 devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 208 | |
| 209 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 210 | iommu = dev->archdata.iommu; |
| 211 | pbm = dev->archdata.host_controller; |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 212 | devhandle = pbm->devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 213 | entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); |
| 214 | |
| 215 | spin_lock_irqsave(&iommu->lock, flags); |
| 216 | |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 217 | iommu_range_free(iommu, dvma, npages); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 218 | |
| 219 | do { |
| 220 | unsigned long num; |
| 221 | |
| 222 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), |
| 223 | npages); |
| 224 | entry += num; |
| 225 | npages -= num; |
| 226 | } while (npages != 0); |
| 227 | |
| 228 | spin_unlock_irqrestore(&iommu->lock, flags); |
| 229 | |
| 230 | order = get_order(size); |
| 231 | if (order < 10) |
| 232 | free_pages((unsigned long)cpu, order); |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 233 | } |
| 234 | |
FUJITA Tomonori | 797a756 | 2009-05-14 16:23:10 +0000 | [diff] [blame] | 235 | static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, |
| 236 | unsigned long offset, size_t sz, |
FUJITA Tomonori | bc0a14f | 2009-08-10 11:53:12 +0900 | [diff] [blame] | 237 | enum dma_data_direction direction, |
| 238 | struct dma_attrs *attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 239 | { |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 240 | struct iommu *iommu; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 241 | unsigned long flags, npages, oaddr; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 242 | unsigned long i, base_paddr; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 243 | u32 bus_addr, ret; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 244 | unsigned long prot; |
| 245 | long entry; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 246 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 247 | iommu = dev->archdata.iommu; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 248 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 249 | if (unlikely(direction == DMA_NONE)) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 250 | goto bad; |
| 251 | |
FUJITA Tomonori | 797a756 | 2009-05-14 16:23:10 +0000 | [diff] [blame] | 252 | oaddr = (unsigned long)(page_address(page) + offset); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 253 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); |
| 254 | npages >>= IO_PAGE_SHIFT; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 255 | |
| 256 | spin_lock_irqsave(&iommu->lock, flags); |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 257 | entry = iommu_range_alloc(dev, iommu, npages, NULL); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 258 | spin_unlock_irqrestore(&iommu->lock, flags); |
| 259 | |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 260 | if (unlikely(entry == DMA_ERROR_CODE)) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 261 | goto bad; |
| 262 | |
| 263 | bus_addr = (iommu->page_table_map_base + |
| 264 | (entry << IO_PAGE_SHIFT)); |
| 265 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); |
| 266 | base_paddr = __pa(oaddr & IO_PAGE_MASK); |
| 267 | prot = HV_PCI_MAP_ATTR_READ; |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 268 | if (direction != DMA_TO_DEVICE) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 269 | prot |= HV_PCI_MAP_ATTR_WRITE; |
| 270 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 271 | local_irq_save(flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 272 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 273 | iommu_batch_start(dev, prot, entry); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 274 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 275 | for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) { |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 276 | long err = iommu_batch_add(base_paddr); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 277 | if (unlikely(err < 0L)) |
| 278 | goto iommu_map_fail; |
| 279 | } |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 280 | if (unlikely(iommu_batch_end() < 0L)) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 281 | goto iommu_map_fail; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 282 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 283 | local_irq_restore(flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 284 | |
| 285 | return ret; |
| 286 | |
| 287 | bad: |
| 288 | if (printk_ratelimit()) |
| 289 | WARN_ON(1); |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 290 | return DMA_ERROR_CODE; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 291 | |
| 292 | iommu_map_fail: |
| 293 | /* Interrupts are disabled. */ |
| 294 | spin_lock(&iommu->lock); |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 295 | iommu_range_free(iommu, bus_addr, npages); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 296 | spin_unlock_irqrestore(&iommu->lock, flags); |
| 297 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 298 | return DMA_ERROR_CODE; |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 299 | } |
| 300 | |
FUJITA Tomonori | 797a756 | 2009-05-14 16:23:10 +0000 | [diff] [blame] | 301 | static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, |
FUJITA Tomonori | bc0a14f | 2009-08-10 11:53:12 +0900 | [diff] [blame] | 302 | size_t sz, enum dma_data_direction direction, |
| 303 | struct dma_attrs *attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 304 | { |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 305 | struct pci_pbm_info *pbm; |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 306 | struct iommu *iommu; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 307 | unsigned long flags, npages; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 308 | long entry; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 309 | u32 devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 310 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 311 | if (unlikely(direction == DMA_NONE)) { |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 312 | if (printk_ratelimit()) |
| 313 | WARN_ON(1); |
| 314 | return; |
| 315 | } |
| 316 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 317 | iommu = dev->archdata.iommu; |
| 318 | pbm = dev->archdata.host_controller; |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 319 | devhandle = pbm->devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 320 | |
| 321 | npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); |
| 322 | npages >>= IO_PAGE_SHIFT; |
| 323 | bus_addr &= IO_PAGE_MASK; |
| 324 | |
| 325 | spin_lock_irqsave(&iommu->lock, flags); |
| 326 | |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 327 | iommu_range_free(iommu, bus_addr, npages); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 328 | |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 329 | entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 330 | do { |
| 331 | unsigned long num; |
| 332 | |
| 333 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), |
| 334 | npages); |
| 335 | entry += num; |
| 336 | npages -= num; |
| 337 | } while (npages != 0); |
| 338 | |
| 339 | spin_unlock_irqrestore(&iommu->lock, flags); |
| 340 | } |
| 341 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 342 | static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, |
FUJITA Tomonori | bc0a14f | 2009-08-10 11:53:12 +0900 | [diff] [blame] | 343 | int nelems, enum dma_data_direction direction, |
| 344 | struct dma_attrs *attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 345 | { |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 346 | struct scatterlist *s, *outs, *segstart; |
| 347 | unsigned long flags, handle, prot; |
| 348 | dma_addr_t dma_next = 0, dma_addr; |
| 349 | unsigned int max_seg_size; |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 350 | unsigned long seg_boundary_size; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 351 | int outcount, incount, i; |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 352 | struct iommu *iommu; |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 353 | unsigned long base_shift; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 354 | long err; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 355 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 356 | BUG_ON(direction == DMA_NONE); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 357 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 358 | iommu = dev->archdata.iommu; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 359 | if (nelems == 0 || !iommu) |
| 360 | return 0; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 361 | |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 362 | prot = HV_PCI_MAP_ATTR_READ; |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 363 | if (direction != DMA_TO_DEVICE) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 364 | prot |= HV_PCI_MAP_ATTR_WRITE; |
| 365 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 366 | outs = s = segstart = &sglist[0]; |
| 367 | outcount = 1; |
| 368 | incount = nelems; |
| 369 | handle = 0; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 370 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 371 | /* Init first segment length for backout at failure */ |
| 372 | outs->dma_length = 0; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 373 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 374 | spin_lock_irqsave(&iommu->lock, flags); |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 375 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 376 | iommu_batch_start(dev, prot, ~0UL); |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 377 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 378 | max_seg_size = dma_get_max_seg_size(dev); |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 379 | seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, |
| 380 | IO_PAGE_SIZE) >> IO_PAGE_SHIFT; |
| 381 | base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 382 | for_each_sg(sglist, s, nelems, i) { |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 383 | unsigned long paddr, npages, entry, out_entry = 0, slen; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 384 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 385 | slen = s->length; |
| 386 | /* Sanity check */ |
| 387 | if (slen == 0) { |
| 388 | dma_next = 0; |
| 389 | continue; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 390 | } |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 391 | /* Allocate iommu entries for that segment */ |
| 392 | paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); |
Joerg Roedel | 0fcff28 | 2008-10-15 22:02:14 -0700 | [diff] [blame] | 393 | npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 394 | entry = iommu_range_alloc(dev, iommu, npages, &handle); |
| 395 | |
| 396 | /* Handle failure */ |
| 397 | if (unlikely(entry == DMA_ERROR_CODE)) { |
| 398 | if (printk_ratelimit()) |
| 399 | printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx" |
| 400 | " npages %lx\n", iommu, paddr, npages); |
| 401 | goto iommu_map_failed; |
| 402 | } |
| 403 | |
| 404 | iommu_batch_new_entry(entry); |
| 405 | |
| 406 | /* Convert entry to a dma_addr_t */ |
| 407 | dma_addr = iommu->page_table_map_base + |
| 408 | (entry << IO_PAGE_SHIFT); |
| 409 | dma_addr |= (s->offset & ~IO_PAGE_MASK); |
| 410 | |
| 411 | /* Insert into HW table */ |
| 412 | paddr &= IO_PAGE_MASK; |
| 413 | while (npages--) { |
| 414 | err = iommu_batch_add(paddr); |
| 415 | if (unlikely(err < 0L)) |
| 416 | goto iommu_map_failed; |
| 417 | paddr += IO_PAGE_SIZE; |
| 418 | } |
| 419 | |
| 420 | /* If we are in an open segment, try merging */ |
| 421 | if (segstart != s) { |
| 422 | /* We cannot merge if: |
| 423 | * - allocated dma_addr isn't contiguous to previous allocation |
| 424 | */ |
| 425 | if ((dma_addr != dma_next) || |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 426 | (outs->dma_length + s->length > max_seg_size) || |
| 427 | (is_span_boundary(out_entry, base_shift, |
| 428 | seg_boundary_size, outs, s))) { |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 429 | /* Can't merge: create a new segment */ |
| 430 | segstart = s; |
| 431 | outcount++; |
| 432 | outs = sg_next(outs); |
| 433 | } else { |
| 434 | outs->dma_length += s->length; |
| 435 | } |
| 436 | } |
| 437 | |
| 438 | if (segstart == s) { |
| 439 | /* This is a new segment, fill entries */ |
| 440 | outs->dma_address = dma_addr; |
| 441 | outs->dma_length = slen; |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 442 | out_entry = entry; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 443 | } |
| 444 | |
| 445 | /* Calculate next page pointer for contiguous check */ |
| 446 | dma_next = dma_addr + slen; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 447 | } |
| 448 | |
| 449 | err = iommu_batch_end(); |
| 450 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 451 | if (unlikely(err < 0L)) |
| 452 | goto iommu_map_failed; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 453 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 454 | spin_unlock_irqrestore(&iommu->lock, flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 455 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 456 | if (outcount < incount) { |
| 457 | outs = sg_next(outs); |
| 458 | outs->dma_address = DMA_ERROR_CODE; |
| 459 | outs->dma_length = 0; |
| 460 | } |
| 461 | |
| 462 | return outcount; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 463 | |
| 464 | iommu_map_failed: |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 465 | for_each_sg(sglist, s, nelems, i) { |
| 466 | if (s->dma_length != 0) { |
| 467 | unsigned long vaddr, npages; |
| 468 | |
| 469 | vaddr = s->dma_address & IO_PAGE_MASK; |
Joerg Roedel | 0fcff28 | 2008-10-15 22:02:14 -0700 | [diff] [blame] | 470 | npages = iommu_num_pages(s->dma_address, s->dma_length, |
| 471 | IO_PAGE_SIZE); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 472 | iommu_range_free(iommu, vaddr, npages); |
| 473 | /* XXX demap? XXX */ |
| 474 | s->dma_address = DMA_ERROR_CODE; |
| 475 | s->dma_length = 0; |
| 476 | } |
| 477 | if (s == outs) |
| 478 | break; |
| 479 | } |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 480 | spin_unlock_irqrestore(&iommu->lock, flags); |
| 481 | |
| 482 | return 0; |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 483 | } |
| 484 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 485 | static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, |
FUJITA Tomonori | bc0a14f | 2009-08-10 11:53:12 +0900 | [diff] [blame] | 486 | int nelems, enum dma_data_direction direction, |
| 487 | struct dma_attrs *attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 488 | { |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 489 | struct pci_pbm_info *pbm; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 490 | struct scatterlist *sg; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 491 | struct iommu *iommu; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 492 | unsigned long flags; |
| 493 | u32 devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 494 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 495 | BUG_ON(direction == DMA_NONE); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 496 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 497 | iommu = dev->archdata.iommu; |
| 498 | pbm = dev->archdata.host_controller; |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 499 | devhandle = pbm->devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 500 | |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 501 | spin_lock_irqsave(&iommu->lock, flags); |
| 502 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 503 | sg = sglist; |
| 504 | while (nelems--) { |
| 505 | dma_addr_t dma_handle = sg->dma_address; |
| 506 | unsigned int len = sg->dma_length; |
| 507 | unsigned long npages, entry; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 508 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 509 | if (!len) |
| 510 | break; |
Joerg Roedel | 0fcff28 | 2008-10-15 22:02:14 -0700 | [diff] [blame] | 511 | npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 512 | iommu_range_free(iommu, dma_handle, npages); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 513 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 514 | entry = ((dma_handle - iommu->page_table_map_base) >> IO_PAGE_SHIFT); |
| 515 | while (npages) { |
| 516 | unsigned long num; |
| 517 | |
| 518 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), |
| 519 | npages); |
| 520 | entry += num; |
| 521 | npages -= num; |
| 522 | } |
| 523 | |
| 524 | sg = sg_next(sg); |
| 525 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 526 | |
| 527 | spin_unlock_irqrestore(&iommu->lock, flags); |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 528 | } |
| 529 | |
FUJITA Tomonori | 02f7a18 | 2009-08-10 11:53:13 +0900 | [diff] [blame] | 530 | static struct dma_map_ops sun4v_dma_ops = { |
Andrzej Pietrasiewicz | c416258 | 2012-03-27 14:56:55 +0200 | [diff] [blame] | 531 | .alloc = dma_4v_alloc_coherent, |
| 532 | .free = dma_4v_free_coherent, |
FUJITA Tomonori | 797a756 | 2009-05-14 16:23:10 +0000 | [diff] [blame] | 533 | .map_page = dma_4v_map_page, |
| 534 | .unmap_page = dma_4v_unmap_page, |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 535 | .map_sg = dma_4v_map_sg, |
| 536 | .unmap_sg = dma_4v_unmap_sg, |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 537 | }; |
| 538 | |
Greg Kroah-Hartman | 7c9503b | 2012-12-21 14:03:26 -0800 | [diff] [blame] | 539 | static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 540 | { |
David S. Miller | e87dc35 | 2006-06-21 18:18:47 -0700 | [diff] [blame] | 541 | struct property *prop; |
| 542 | struct device_node *dp; |
| 543 | |
Grant Likely | 61c7a08 | 2010-04-13 16:12:29 -0700 | [diff] [blame] | 544 | dp = pbm->op->dev.of_node; |
David S. Miller | 34768bc | 2007-05-07 23:06:27 -0700 | [diff] [blame] | 545 | prop = of_find_property(dp, "66mhz-capable", NULL); |
| 546 | pbm->is_66mhz_capable = (prop != NULL); |
David S. Miller | e822358a | 2008-09-01 18:32:22 -0700 | [diff] [blame] | 547 | pbm->pci_bus = pci_scan_one_pbm(pbm, parent); |
David S. Miller | c260926 | 2006-02-12 22:18:52 -0800 | [diff] [blame] | 548 | |
| 549 | /* XXX register error interrupt handlers XXX */ |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 550 | } |
| 551 | |
Greg Kroah-Hartman | 7c9503b | 2012-12-21 14:03:26 -0800 | [diff] [blame] | 552 | static unsigned long probe_existing_entries(struct pci_pbm_info *pbm, |
| 553 | struct iommu *iommu) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 554 | { |
David S. Miller | 9b3627f | 2007-04-24 23:51:18 -0700 | [diff] [blame] | 555 | struct iommu_arena *arena = &iommu->arena; |
David S. Miller | e7a0453 | 2006-02-15 22:25:27 -0800 | [diff] [blame] | 556 | unsigned long i, cnt = 0; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 557 | u32 devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 558 | |
| 559 | devhandle = pbm->devhandle; |
| 560 | for (i = 0; i < arena->limit; i++) { |
| 561 | unsigned long ret, io_attrs, ra; |
| 562 | |
| 563 | ret = pci_sun4v_iommu_getmap(devhandle, |
| 564 | HV_PCI_TSBID(0, i), |
| 565 | &io_attrs, &ra); |
David S. Miller | e7a0453 | 2006-02-15 22:25:27 -0800 | [diff] [blame] | 566 | if (ret == HV_EOK) { |
David S. Miller | c2a5a46 | 2006-06-22 00:01:56 -0700 | [diff] [blame] | 567 | if (page_in_phys_avail(ra)) { |
| 568 | pci_sun4v_iommu_demap(devhandle, |
| 569 | HV_PCI_TSBID(0, i), 1); |
| 570 | } else { |
| 571 | cnt++; |
| 572 | __set_bit(i, arena->map); |
| 573 | } |
David S. Miller | e7a0453 | 2006-02-15 22:25:27 -0800 | [diff] [blame] | 574 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 575 | } |
David S. Miller | e7a0453 | 2006-02-15 22:25:27 -0800 | [diff] [blame] | 576 | |
| 577 | return cnt; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 578 | } |
| 579 | |
Greg Kroah-Hartman | 7c9503b | 2012-12-21 14:03:26 -0800 | [diff] [blame] | 580 | static int pci_sun4v_iommu_init(struct pci_pbm_info *pbm) |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 581 | { |
David S. Miller | 8aef727 | 2008-09-01 20:23:18 -0700 | [diff] [blame] | 582 | static const u32 vdma_default[] = { 0x80000000, 0x80000000 }; |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 583 | struct iommu *iommu = pbm->iommu; |
David S. Miller | c6fee08 | 2011-02-26 23:40:02 -0800 | [diff] [blame] | 584 | unsigned long num_tsb_entries, sz; |
David S. Miller | 8aef727 | 2008-09-01 20:23:18 -0700 | [diff] [blame] | 585 | u32 dma_mask, dma_offset; |
| 586 | const u32 *vdma; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 587 | |
Grant Likely | 61c7a08 | 2010-04-13 16:12:29 -0700 | [diff] [blame] | 588 | vdma = of_get_property(pbm->op->dev.of_node, "virtual-dma", NULL); |
David S. Miller | 8aef727 | 2008-09-01 20:23:18 -0700 | [diff] [blame] | 589 | if (!vdma) |
| 590 | vdma = vdma_default; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 591 | |
David S. Miller | 59db810 | 2007-05-23 18:00:46 -0700 | [diff] [blame] | 592 | if ((vdma[0] | vdma[1]) & ~IO_PAGE_MASK) { |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 593 | printk(KERN_ERR PFX "Strange virtual-dma[%08x:%08x].\n", |
| 594 | vdma[0], vdma[1]); |
| 595 | return -EINVAL; |
Peter Senna Tschudin | 20b739f | 2012-09-12 07:03:11 +0000 | [diff] [blame] | 596 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 597 | |
David S. Miller | 59db810 | 2007-05-23 18:00:46 -0700 | [diff] [blame] | 598 | dma_mask = (roundup_pow_of_two(vdma[1]) - 1UL); |
| 599 | num_tsb_entries = vdma[1] / IO_PAGE_SIZE; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 600 | |
| 601 | dma_offset = vdma[0]; |
| 602 | |
| 603 | /* Setup initial software IOMMU state. */ |
| 604 | spin_lock_init(&iommu->lock); |
| 605 | iommu->ctx_lowest_free = 1; |
| 606 | iommu->page_table_map_base = dma_offset; |
| 607 | iommu->dma_addr_mask = dma_mask; |
| 608 | |
| 609 | /* Allocate and initialize the free area map. */ |
David S. Miller | 59db810 | 2007-05-23 18:00:46 -0700 | [diff] [blame] | 610 | sz = (num_tsb_entries + 7) / 8; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 611 | sz = (sz + 7UL) & ~7UL; |
Yan Burman | 982c206 | 2006-11-30 17:13:09 -0800 | [diff] [blame] | 612 | iommu->arena.map = kzalloc(sz, GFP_KERNEL); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 613 | if (!iommu->arena.map) { |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 614 | printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); |
| 615 | return -ENOMEM; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 616 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 617 | iommu->arena.limit = num_tsb_entries; |
| 618 | |
David S. Miller | e7a0453 | 2006-02-15 22:25:27 -0800 | [diff] [blame] | 619 | sz = probe_existing_entries(pbm, iommu); |
David S. Miller | c2a5a46 | 2006-06-22 00:01:56 -0700 | [diff] [blame] | 620 | if (sz) |
| 621 | printk("%s: Imported %lu TSB entries from OBP\n", |
| 622 | pbm->name, sz); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 623 | |
| 624 | return 0; |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 625 | } |
| 626 | |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 627 | #ifdef CONFIG_PCI_MSI |
| 628 | struct pci_sun4v_msiq_entry { |
| 629 | u64 version_type; |
| 630 | #define MSIQ_VERSION_MASK 0xffffffff00000000UL |
| 631 | #define MSIQ_VERSION_SHIFT 32 |
| 632 | #define MSIQ_TYPE_MASK 0x00000000000000ffUL |
| 633 | #define MSIQ_TYPE_SHIFT 0 |
| 634 | #define MSIQ_TYPE_NONE 0x00 |
| 635 | #define MSIQ_TYPE_MSG 0x01 |
| 636 | #define MSIQ_TYPE_MSI32 0x02 |
| 637 | #define MSIQ_TYPE_MSI64 0x03 |
| 638 | #define MSIQ_TYPE_INTX 0x08 |
| 639 | #define MSIQ_TYPE_NONE2 0xff |
| 640 | |
| 641 | u64 intx_sysino; |
| 642 | u64 reserved1; |
| 643 | u64 stick; |
| 644 | u64 req_id; /* bus/device/func */ |
| 645 | #define MSIQ_REQID_BUS_MASK 0xff00UL |
| 646 | #define MSIQ_REQID_BUS_SHIFT 8 |
| 647 | #define MSIQ_REQID_DEVICE_MASK 0x00f8UL |
| 648 | #define MSIQ_REQID_DEVICE_SHIFT 3 |
| 649 | #define MSIQ_REQID_FUNC_MASK 0x0007UL |
| 650 | #define MSIQ_REQID_FUNC_SHIFT 0 |
| 651 | |
| 652 | u64 msi_address; |
| 653 | |
Simon Arlott | e5dd42e | 2007-05-11 13:52:08 -0700 | [diff] [blame] | 654 | /* The format of this value is message type dependent. |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 655 | * For MSI bits 15:0 are the data from the MSI packet. |
| 656 | * For MSI-X bits 31:0 are the data from the MSI packet. |
| 657 | * For MSG, the message code and message routing code where: |
| 658 | * bits 39:32 is the bus/device/fn of the msg target-id |
| 659 | * bits 18:16 is the message routing code |
| 660 | * bits 7:0 is the message code |
| 661 | * For INTx the low order 2-bits are: |
| 662 | * 00 - INTA |
| 663 | * 01 - INTB |
| 664 | * 10 - INTC |
| 665 | * 11 - INTD |
| 666 | */ |
| 667 | u64 msi_data; |
| 668 | |
| 669 | u64 reserved2; |
| 670 | }; |
| 671 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 672 | static int pci_sun4v_get_head(struct pci_pbm_info *pbm, unsigned long msiqid, |
| 673 | unsigned long *head) |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 674 | { |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 675 | unsigned long err, limit; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 676 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 677 | err = pci_sun4v_msiq_gethead(pbm->devhandle, msiqid, head); |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 678 | if (unlikely(err)) |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 679 | return -ENXIO; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 680 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 681 | limit = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); |
| 682 | if (unlikely(*head >= limit)) |
| 683 | return -EFBIG; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 684 | |
| 685 | return 0; |
| 686 | } |
| 687 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 688 | static int pci_sun4v_dequeue_msi(struct pci_pbm_info *pbm, |
| 689 | unsigned long msiqid, unsigned long *head, |
| 690 | unsigned long *msi) |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 691 | { |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 692 | struct pci_sun4v_msiq_entry *ep; |
| 693 | unsigned long err, type; |
| 694 | |
| 695 | /* Note: void pointer arithmetic, 'head' is a byte offset */ |
| 696 | ep = (pbm->msi_queues + ((msiqid - pbm->msiq_first) * |
| 697 | (pbm->msiq_ent_count * |
| 698 | sizeof(struct pci_sun4v_msiq_entry))) + |
| 699 | *head); |
| 700 | |
| 701 | if ((ep->version_type & MSIQ_TYPE_MASK) == 0) |
| 702 | return 0; |
| 703 | |
| 704 | type = (ep->version_type & MSIQ_TYPE_MASK) >> MSIQ_TYPE_SHIFT; |
| 705 | if (unlikely(type != MSIQ_TYPE_MSI32 && |
| 706 | type != MSIQ_TYPE_MSI64)) |
| 707 | return -EINVAL; |
| 708 | |
| 709 | *msi = ep->msi_data; |
| 710 | |
| 711 | err = pci_sun4v_msi_setstate(pbm->devhandle, |
| 712 | ep->msi_data /* msi_num */, |
| 713 | HV_MSISTATE_IDLE); |
| 714 | if (unlikely(err)) |
| 715 | return -ENXIO; |
| 716 | |
| 717 | /* Clear the entry. */ |
| 718 | ep->version_type &= ~MSIQ_TYPE_MASK; |
| 719 | |
| 720 | (*head) += sizeof(struct pci_sun4v_msiq_entry); |
| 721 | if (*head >= |
| 722 | (pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry))) |
| 723 | *head = 0; |
| 724 | |
| 725 | return 1; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 726 | } |
| 727 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 728 | static int pci_sun4v_set_head(struct pci_pbm_info *pbm, unsigned long msiqid, |
| 729 | unsigned long head) |
| 730 | { |
| 731 | unsigned long err; |
| 732 | |
| 733 | err = pci_sun4v_msiq_sethead(pbm->devhandle, msiqid, head); |
| 734 | if (unlikely(err)) |
| 735 | return -EINVAL; |
| 736 | |
| 737 | return 0; |
| 738 | } |
| 739 | |
| 740 | static int pci_sun4v_msi_setup(struct pci_pbm_info *pbm, unsigned long msiqid, |
| 741 | unsigned long msi, int is_msi64) |
| 742 | { |
| 743 | if (pci_sun4v_msi_setmsiq(pbm->devhandle, msi, msiqid, |
| 744 | (is_msi64 ? |
| 745 | HV_MSITYPE_MSI64 : HV_MSITYPE_MSI32))) |
| 746 | return -ENXIO; |
| 747 | if (pci_sun4v_msi_setstate(pbm->devhandle, msi, HV_MSISTATE_IDLE)) |
| 748 | return -ENXIO; |
| 749 | if (pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_VALID)) |
| 750 | return -ENXIO; |
| 751 | return 0; |
| 752 | } |
| 753 | |
| 754 | static int pci_sun4v_msi_teardown(struct pci_pbm_info *pbm, unsigned long msi) |
| 755 | { |
| 756 | unsigned long err, msiqid; |
| 757 | |
| 758 | err = pci_sun4v_msi_getmsiq(pbm->devhandle, msi, &msiqid); |
| 759 | if (err) |
| 760 | return -ENXIO; |
| 761 | |
| 762 | pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_INVALID); |
| 763 | |
| 764 | return 0; |
| 765 | } |
| 766 | |
| 767 | static int pci_sun4v_msiq_alloc(struct pci_pbm_info *pbm) |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 768 | { |
| 769 | unsigned long q_size, alloc_size, pages, order; |
| 770 | int i; |
| 771 | |
| 772 | q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); |
| 773 | alloc_size = (pbm->msiq_num * q_size); |
| 774 | order = get_order(alloc_size); |
| 775 | pages = __get_free_pages(GFP_KERNEL | __GFP_COMP, order); |
| 776 | if (pages == 0UL) { |
| 777 | printk(KERN_ERR "MSI: Cannot allocate MSI queues (o=%lu).\n", |
| 778 | order); |
| 779 | return -ENOMEM; |
| 780 | } |
| 781 | memset((char *)pages, 0, PAGE_SIZE << order); |
| 782 | pbm->msi_queues = (void *) pages; |
| 783 | |
| 784 | for (i = 0; i < pbm->msiq_num; i++) { |
| 785 | unsigned long err, base = __pa(pages + (i * q_size)); |
| 786 | unsigned long ret1, ret2; |
| 787 | |
| 788 | err = pci_sun4v_msiq_conf(pbm->devhandle, |
| 789 | pbm->msiq_first + i, |
| 790 | base, pbm->msiq_ent_count); |
| 791 | if (err) { |
| 792 | printk(KERN_ERR "MSI: msiq register fails (err=%lu)\n", |
| 793 | err); |
| 794 | goto h_error; |
| 795 | } |
| 796 | |
| 797 | err = pci_sun4v_msiq_info(pbm->devhandle, |
| 798 | pbm->msiq_first + i, |
| 799 | &ret1, &ret2); |
| 800 | if (err) { |
| 801 | printk(KERN_ERR "MSI: Cannot read msiq (err=%lu)\n", |
| 802 | err); |
| 803 | goto h_error; |
| 804 | } |
| 805 | if (ret1 != base || ret2 != pbm->msiq_ent_count) { |
| 806 | printk(KERN_ERR "MSI: Bogus qconf " |
| 807 | "expected[%lx:%x] got[%lx:%lx]\n", |
| 808 | base, pbm->msiq_ent_count, |
| 809 | ret1, ret2); |
| 810 | goto h_error; |
| 811 | } |
| 812 | } |
| 813 | |
| 814 | return 0; |
| 815 | |
| 816 | h_error: |
| 817 | free_pages(pages, order); |
| 818 | return -EINVAL; |
| 819 | } |
| 820 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 821 | static void pci_sun4v_msiq_free(struct pci_pbm_info *pbm) |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 822 | { |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 823 | unsigned long q_size, alloc_size, pages, order; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 824 | int i; |
| 825 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 826 | for (i = 0; i < pbm->msiq_num; i++) { |
| 827 | unsigned long msiqid = pbm->msiq_first + i; |
| 828 | |
| 829 | (void) pci_sun4v_msiq_conf(pbm->devhandle, msiqid, 0UL, 0); |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 830 | } |
| 831 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 832 | q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); |
| 833 | alloc_size = (pbm->msiq_num * q_size); |
| 834 | order = get_order(alloc_size); |
| 835 | |
| 836 | pages = (unsigned long) pbm->msi_queues; |
| 837 | |
| 838 | free_pages(pages, order); |
| 839 | |
| 840 | pbm->msi_queues = NULL; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 841 | } |
| 842 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 843 | static int pci_sun4v_msiq_build_irq(struct pci_pbm_info *pbm, |
| 844 | unsigned long msiqid, |
| 845 | unsigned long devino) |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 846 | { |
Sam Ravnborg | 44ed3c0 | 2011-01-22 11:32:20 +0000 | [diff] [blame] | 847 | unsigned int irq = sun4v_build_irq(pbm->devhandle, devino); |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 848 | |
Sam Ravnborg | 44ed3c0 | 2011-01-22 11:32:20 +0000 | [diff] [blame] | 849 | if (!irq) |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 850 | return -ENOMEM; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 851 | |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 852 | if (pci_sun4v_msiq_setvalid(pbm->devhandle, msiqid, HV_MSIQ_VALID)) |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 853 | return -EINVAL; |
David S. Miller | 7cc8583 | 2011-12-22 13:23:59 -0800 | [diff] [blame] | 854 | if (pci_sun4v_msiq_setstate(pbm->devhandle, msiqid, HV_MSIQSTATE_IDLE)) |
| 855 | return -EINVAL; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 856 | |
Sam Ravnborg | 44ed3c0 | 2011-01-22 11:32:20 +0000 | [diff] [blame] | 857 | return irq; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 858 | } |
| 859 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 860 | static const struct sparc64_msiq_ops pci_sun4v_msiq_ops = { |
| 861 | .get_head = pci_sun4v_get_head, |
| 862 | .dequeue_msi = pci_sun4v_dequeue_msi, |
| 863 | .set_head = pci_sun4v_set_head, |
| 864 | .msi_setup = pci_sun4v_msi_setup, |
| 865 | .msi_teardown = pci_sun4v_msi_teardown, |
| 866 | .msiq_alloc = pci_sun4v_msiq_alloc, |
| 867 | .msiq_free = pci_sun4v_msiq_free, |
| 868 | .msiq_build_irq = pci_sun4v_msiq_build_irq, |
| 869 | }; |
David S. Miller | e9870c4 | 2007-05-07 23:28:50 -0700 | [diff] [blame] | 870 | |
| 871 | static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) |
| 872 | { |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 873 | sparc64_pbm_msi_init(pbm, &pci_sun4v_msiq_ops); |
David S. Miller | e9870c4 | 2007-05-07 23:28:50 -0700 | [diff] [blame] | 874 | } |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 875 | #else /* CONFIG_PCI_MSI */ |
| 876 | static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) |
| 877 | { |
| 878 | } |
| 879 | #endif /* !(CONFIG_PCI_MSI) */ |
| 880 | |
Greg Kroah-Hartman | 7c9503b | 2012-12-21 14:03:26 -0800 | [diff] [blame] | 881 | static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm, |
| 882 | struct platform_device *op, u32 devhandle) |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 883 | { |
Grant Likely | 61c7a08 | 2010-04-13 16:12:29 -0700 | [diff] [blame] | 884 | struct device_node *dp = op->dev.of_node; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 885 | int err; |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 886 | |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 887 | pbm->numa_node = of_node_to_nid(dp); |
| 888 | |
David S. Miller | ca3dd88 | 2007-05-09 02:35:27 -0700 | [diff] [blame] | 889 | pbm->pci_ops = &sun4v_pci_ops; |
| 890 | pbm->config_space_reg_bits = 12; |
David S. Miller | 34768bc | 2007-05-07 23:06:27 -0700 | [diff] [blame] | 891 | |
David S. Miller | 6c108f1 | 2007-05-07 23:49:01 -0700 | [diff] [blame] | 892 | pbm->index = pci_num_pbms++; |
| 893 | |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 894 | pbm->op = op; |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 895 | |
David S. Miller | 3833789 | 2006-02-12 22:06:53 -0800 | [diff] [blame] | 896 | pbm->devhandle = devhandle; |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 897 | |
David S. Miller | e87dc35 | 2006-06-21 18:18:47 -0700 | [diff] [blame] | 898 | pbm->name = dp->full_name; |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 899 | |
David S. Miller | e87dc35 | 2006-06-21 18:18:47 -0700 | [diff] [blame] | 900 | printk("%s: SUN4V PCI Bus Module\n", pbm->name); |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 901 | printk("%s: On NUMA node %d\n", pbm->name, pbm->numa_node); |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 902 | |
David S. Miller | 9fd8b64 | 2007-03-08 21:55:49 -0800 | [diff] [blame] | 903 | pci_determine_mem_io_space(pbm); |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 904 | |
David S. Miller | cfa0652 | 2007-05-07 21:51:41 -0700 | [diff] [blame] | 905 | pci_get_pbm_props(pbm); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 906 | |
| 907 | err = pci_sun4v_iommu_init(pbm); |
| 908 | if (err) |
| 909 | return err; |
| 910 | |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 911 | pci_sun4v_msi_init(pbm); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 912 | |
David S. Miller | e822358a | 2008-09-01 18:32:22 -0700 | [diff] [blame] | 913 | pci_sun4v_scan_bus(pbm, &op->dev); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 914 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 915 | pbm->next = pci_pbm_root; |
| 916 | pci_pbm_root = pbm; |
| 917 | |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 918 | return 0; |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 919 | } |
| 920 | |
Greg Kroah-Hartman | 7c9503b | 2012-12-21 14:03:26 -0800 | [diff] [blame] | 921 | static int pci_sun4v_probe(struct platform_device *op) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 922 | { |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 923 | const struct linux_prom64_registers *regs; |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 924 | static int hvapi_negotiated = 0; |
David S. Miller | 34768bc | 2007-05-07 23:06:27 -0700 | [diff] [blame] | 925 | struct pci_pbm_info *pbm; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 926 | struct device_node *dp; |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 927 | struct iommu *iommu; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 928 | u32 devhandle; |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 929 | int i, err; |
David S. Miller | 3833789 | 2006-02-12 22:06:53 -0800 | [diff] [blame] | 930 | |
Grant Likely | 61c7a08 | 2010-04-13 16:12:29 -0700 | [diff] [blame] | 931 | dp = op->dev.of_node; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 932 | |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 933 | if (!hvapi_negotiated++) { |
David S. Miller | 8d2aec5 | 2008-09-12 00:01:03 -0700 | [diff] [blame] | 934 | err = sun4v_hvapi_register(HV_GRP_PCI, |
| 935 | vpci_major, |
| 936 | &vpci_minor); |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 937 | |
| 938 | if (err) { |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 939 | printk(KERN_ERR PFX "Could not register hvapi, " |
| 940 | "err=%d\n", err); |
| 941 | return err; |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 942 | } |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 943 | printk(KERN_INFO PFX "Registered hvapi major[%lu] minor[%lu]\n", |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 944 | vpci_major, vpci_minor); |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 945 | |
| 946 | dma_ops = &sun4v_dma_ops; |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 947 | } |
| 948 | |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 949 | regs = of_get_property(dp, "reg", NULL); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 950 | err = -ENODEV; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 951 | if (!regs) { |
| 952 | printk(KERN_ERR PFX "Could not find config registers\n"); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 953 | goto out_err; |
Cyrill Gorcunov | 75c6d14 | 2007-11-20 17:32:19 -0800 | [diff] [blame] | 954 | } |
David S. Miller | e87dc35 | 2006-06-21 18:18:47 -0700 | [diff] [blame] | 955 | devhandle = (regs->phys_addr >> 32UL) & 0x0fffffff; |
David S. Miller | 3833789 | 2006-02-12 22:06:53 -0800 | [diff] [blame] | 956 | |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 957 | err = -ENOMEM; |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 958 | if (!iommu_batch_initialized) { |
| 959 | for_each_possible_cpu(i) { |
| 960 | unsigned long page = get_zeroed_page(GFP_KERNEL); |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 961 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 962 | if (!page) |
| 963 | goto out_err; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 964 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 965 | per_cpu(iommu_batch, i).pglist = (u64 *) page; |
| 966 | } |
| 967 | iommu_batch_initialized = 1; |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 968 | } |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 969 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 970 | pbm = kzalloc(sizeof(*pbm), GFP_KERNEL); |
| 971 | if (!pbm) { |
| 972 | printk(KERN_ERR PFX "Could not allocate pci_pbm_info\n"); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 973 | goto out_err; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 974 | } |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 975 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 976 | iommu = kzalloc(sizeof(struct iommu), GFP_KERNEL); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 977 | if (!iommu) { |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 978 | printk(KERN_ERR PFX "Could not allocate pbm iommu\n"); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 979 | goto out_free_controller; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 980 | } |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 981 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 982 | pbm->iommu = iommu; |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 983 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 984 | err = pci_sun4v_pbm_init(pbm, op, devhandle); |
| 985 | if (err) |
| 986 | goto out_free_iommu; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 987 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 988 | dev_set_drvdata(&op->dev, pbm); |
David S. Miller | bade5622 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 989 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 990 | return 0; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 991 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 992 | out_free_iommu: |
| 993 | kfree(pbm->iommu); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 994 | |
| 995 | out_free_controller: |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 996 | kfree(pbm); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 997 | |
| 998 | out_err: |
| 999 | return err; |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 1000 | } |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1001 | |
David S. Miller | 3628aa0 | 2011-03-30 17:37:56 -0700 | [diff] [blame] | 1002 | static const struct of_device_id pci_sun4v_match[] = { |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1003 | { |
| 1004 | .name = "pci", |
| 1005 | .compatible = "SUNW,sun4v-pci", |
| 1006 | }, |
| 1007 | {}, |
| 1008 | }; |
| 1009 | |
Grant Likely | 4ebb24f | 2011-02-22 20:01:33 -0700 | [diff] [blame] | 1010 | static struct platform_driver pci_sun4v_driver = { |
Grant Likely | 4018294 | 2010-04-13 16:13:02 -0700 | [diff] [blame] | 1011 | .driver = { |
| 1012 | .name = DRIVER_NAME, |
| 1013 | .owner = THIS_MODULE, |
| 1014 | .of_match_table = pci_sun4v_match, |
| 1015 | }, |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1016 | .probe = pci_sun4v_probe, |
| 1017 | }; |
| 1018 | |
| 1019 | static int __init pci_sun4v_init(void) |
| 1020 | { |
Grant Likely | 4ebb24f | 2011-02-22 20:01:33 -0700 | [diff] [blame] | 1021 | return platform_driver_register(&pci_sun4v_driver); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1022 | } |
| 1023 | |
| 1024 | subsys_initcall(pci_sun4v_init); |