blob: 027c9be976544edc7029431e04e5f9b8697503ed [file] [log] [blame]
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +00001/*
2 * Renesas SuperH DMA Engine support
3 *
4 * base is drivers/dma/flsdma.c
5 *
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +02006 * Copyright (C) 2011-2012 Guennadi Liakhovetski <g.liakhovetski@gmx.de>
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +00007 * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>
8 * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved.
9 * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
10 *
11 * This is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * - DMA of SuperH does not have Hardware DMA chain mode.
17 * - MAX DMA size is 16MB.
18 *
19 */
20
21#include <linux/init.h>
22#include <linux/module.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090023#include <linux/slab.h>
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +000024#include <linux/interrupt.h>
25#include <linux/dmaengine.h>
26#include <linux/delay.h>
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +000027#include <linux/platform_device.h>
Guennadi Liakhovetski20f2a3b2010-02-11 16:50:18 +000028#include <linux/pm_runtime.h>
Magnus Dammb2623a62010-03-19 04:47:10 +000029#include <linux/sh_dma.h>
Paul Mundt03aa18f2010-12-17 19:16:10 +090030#include <linux/notifier.h>
31#include <linux/kdebug.h>
32#include <linux/spinlock.h>
33#include <linux/rculist.h>
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +000034
Guennadi Liakhovetskie95be942012-07-02 22:30:53 +020035#include "../dmaengine.h"
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +000036#include "shdma.h"
37
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +020038#define SH_DMAE_DRV_NAME "sh-dma-engine"
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +000039
Guennadi Liakhovetski8b1935e2010-02-11 16:50:14 +000040/* Default MEMCPY transfer size = 2^2 = 4 bytes */
41#define LOG2_DEFAULT_XFER_SIZE 2
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +020042#define SH_DMA_SLAVE_NUMBER 256
43#define SH_DMA_TCR_MAX (16 * 1024 * 1024 - 1)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +000044
Paul Mundt03aa18f2010-12-17 19:16:10 +090045/*
46 * Used for write-side mutual exclusion for the global device list,
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +000047 * read-side synchronization by way of RCU, and per-controller data.
Paul Mundt03aa18f2010-12-17 19:16:10 +090048 */
49static DEFINE_SPINLOCK(sh_dmae_lock);
50static LIST_HEAD(sh_dmae_devices);
51
Guennadi Liakhovetskic11b46c322012-01-04 15:34:17 +010052static void chclr_write(struct sh_dmae_chan *sh_dc, u32 data)
53{
54 struct sh_dmae_device *shdev = to_sh_dev(sh_dc);
55
56 __raw_writel(data, shdev->chan_reg +
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +020057 shdev->pdata->channel[sh_dc->shdma_chan.id].chclr_offset);
Guennadi Liakhovetskic11b46c322012-01-04 15:34:17 +010058}
Guennadi Liakhovetski3542a112009-12-17 09:41:39 -070059
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +000060static void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg)
61{
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +000062 __raw_writel(data, sh_dc->base + reg / sizeof(u32));
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +000063}
64
65static u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg)
66{
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +000067 return __raw_readl(sh_dc->base + reg / sizeof(u32));
68}
69
70static u16 dmaor_read(struct sh_dmae_device *shdev)
71{
Kuninori Morimotoe76c3af2011-06-17 08:20:56 +000072 u32 __iomem *addr = shdev->chan_reg + DMAOR / sizeof(u32);
73
74 if (shdev->pdata->dmaor_is_32bit)
75 return __raw_readl(addr);
76 else
77 return __raw_readw(addr);
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +000078}
79
80static void dmaor_write(struct sh_dmae_device *shdev, u16 data)
81{
Kuninori Morimotoe76c3af2011-06-17 08:20:56 +000082 u32 __iomem *addr = shdev->chan_reg + DMAOR / sizeof(u32);
83
84 if (shdev->pdata->dmaor_is_32bit)
85 __raw_writel(data, addr);
86 else
87 __raw_writew(data, addr);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +000088}
89
Kuninori Morimoto5899a722011-06-17 08:20:40 +000090static void chcr_write(struct sh_dmae_chan *sh_dc, u32 data)
91{
92 struct sh_dmae_device *shdev = to_sh_dev(sh_dc);
93
94 __raw_writel(data, sh_dc->base + shdev->chcr_offset / sizeof(u32));
95}
96
97static u32 chcr_read(struct sh_dmae_chan *sh_dc)
98{
99 struct sh_dmae_device *shdev = to_sh_dev(sh_dc);
100
101 return __raw_readl(sh_dc->base + shdev->chcr_offset / sizeof(u32));
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000102}
103
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000104/*
105 * Reset DMA controller
106 *
107 * SH7780 has two DMAOR register
108 */
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000109static void sh_dmae_ctl_stop(struct sh_dmae_device *shdev)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000110{
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000111 unsigned short dmaor;
112 unsigned long flags;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000113
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000114 spin_lock_irqsave(&sh_dmae_lock, flags);
115
116 dmaor = dmaor_read(shdev);
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000117 dmaor_write(shdev, dmaor & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME));
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000118
119 spin_unlock_irqrestore(&sh_dmae_lock, flags);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000120}
121
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000122static int sh_dmae_rst(struct sh_dmae_device *shdev)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000123{
124 unsigned short dmaor;
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000125 unsigned long flags;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000126
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000127 spin_lock_irqsave(&sh_dmae_lock, flags);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000128
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000129 dmaor = dmaor_read(shdev) & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME);
130
Guennadi Liakhovetskic11b46c322012-01-04 15:34:17 +0100131 if (shdev->pdata->chclr_present) {
132 int i;
133 for (i = 0; i < shdev->pdata->channel_num; i++) {
134 struct sh_dmae_chan *sh_chan = shdev->chan[i];
135 if (sh_chan)
136 chclr_write(sh_chan, 0);
137 }
138 }
139
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000140 dmaor_write(shdev, dmaor | shdev->pdata->dmaor_init);
141
142 dmaor = dmaor_read(shdev);
143
144 spin_unlock_irqrestore(&sh_dmae_lock, flags);
145
146 if (dmaor & (DMAOR_AE | DMAOR_NMIF)) {
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200147 dev_warn(shdev->shdma_dev.dma_dev.dev, "Can't initialize DMAOR.\n");
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000148 return -EIO;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000149 }
Guennadi Liakhovetskic11b46c322012-01-04 15:34:17 +0100150 if (shdev->pdata->dmaor_init & ~dmaor)
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200151 dev_warn(shdev->shdma_dev.dma_dev.dev,
Guennadi Liakhovetskic11b46c322012-01-04 15:34:17 +0100152 "DMAOR=0x%x hasn't latched the initial value 0x%x.\n",
153 dmaor, shdev->pdata->dmaor_init);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000154 return 0;
155}
156
Guennadi Liakhovetskifc461852010-01-19 07:24:55 +0000157static bool dmae_is_busy(struct sh_dmae_chan *sh_chan)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000158{
Kuninori Morimoto5899a722011-06-17 08:20:40 +0000159 u32 chcr = chcr_read(sh_chan);
Guennadi Liakhovetskifc461852010-01-19 07:24:55 +0000160
161 if ((chcr & (CHCR_DE | CHCR_TE)) == CHCR_DE)
162 return true; /* working */
163
164 return false; /* waiting */
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000165}
166
Guennadi Liakhovetski8b1935e2010-02-11 16:50:14 +0000167static unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan, u32 chcr)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000168{
Kuninori Morimotoc4e0dd72011-06-16 05:08:09 +0000169 struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
Guennadi Liakhovetski8b1935e2010-02-11 16:50:14 +0000170 struct sh_dmae_pdata *pdata = shdev->pdata;
171 int cnt = ((chcr & pdata->ts_low_mask) >> pdata->ts_low_shift) |
172 ((chcr & pdata->ts_high_mask) >> pdata->ts_high_shift);
Guennadi Liakhovetski623b4ac2010-02-03 14:44:12 +0000173
Guennadi Liakhovetski8b1935e2010-02-11 16:50:14 +0000174 if (cnt >= pdata->ts_shift_num)
175 cnt = 0;
176
177 return pdata->ts_shift[cnt];
178}
179
180static u32 log2size_to_chcr(struct sh_dmae_chan *sh_chan, int l2size)
181{
Kuninori Morimotoc4e0dd72011-06-16 05:08:09 +0000182 struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
Guennadi Liakhovetski8b1935e2010-02-11 16:50:14 +0000183 struct sh_dmae_pdata *pdata = shdev->pdata;
184 int i;
185
186 for (i = 0; i < pdata->ts_shift_num; i++)
187 if (pdata->ts_shift[i] == l2size)
188 break;
189
190 if (i == pdata->ts_shift_num)
191 i = 0;
192
193 return ((i << pdata->ts_low_shift) & pdata->ts_low_mask) |
194 ((i << pdata->ts_high_shift) & pdata->ts_high_mask);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000195}
196
Guennadi Liakhovetski3542a112009-12-17 09:41:39 -0700197static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs *hw)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000198{
Guennadi Liakhovetski3542a112009-12-17 09:41:39 -0700199 sh_dmae_writel(sh_chan, hw->sar, SAR);
200 sh_dmae_writel(sh_chan, hw->dar, DAR);
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000201 sh_dmae_writel(sh_chan, hw->tcr >> sh_chan->xmit_shift, TCR);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000202}
203
204static void dmae_start(struct sh_dmae_chan *sh_chan)
205{
Kuninori Morimoto67c62692011-06-17 08:20:51 +0000206 struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
Kuninori Morimoto5899a722011-06-17 08:20:40 +0000207 u32 chcr = chcr_read(sh_chan);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000208
Kuninori Morimoto260bf2c2011-06-17 08:21:05 +0000209 if (shdev->pdata->needs_tend_set)
210 sh_dmae_writel(sh_chan, 0xFFFFFFFF, TEND);
211
Kuninori Morimoto67c62692011-06-17 08:20:51 +0000212 chcr |= CHCR_DE | shdev->chcr_ie_bit;
Kuninori Morimoto5899a722011-06-17 08:20:40 +0000213 chcr_write(sh_chan, chcr & ~CHCR_TE);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000214}
215
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000216static void dmae_init(struct sh_dmae_chan *sh_chan)
217{
Guennadi Liakhovetski8b1935e2010-02-11 16:50:14 +0000218 /*
219 * Default configuration for dual address memory-memory transfer.
220 * 0x400 represents auto-request.
221 */
222 u32 chcr = DM_INC | SM_INC | 0x400 | log2size_to_chcr(sh_chan,
223 LOG2_DEFAULT_XFER_SIZE);
224 sh_chan->xmit_shift = calc_xmit_shift(sh_chan, chcr);
Kuninori Morimoto5899a722011-06-17 08:20:40 +0000225 chcr_write(sh_chan, chcr);
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000226}
227
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000228static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val)
229{
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000230 /* If DMA is active, cannot set CHCR. TODO: remove this superfluous check */
Guennadi Liakhovetskifc461852010-01-19 07:24:55 +0000231 if (dmae_is_busy(sh_chan))
232 return -EBUSY;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000233
Guennadi Liakhovetski8b1935e2010-02-11 16:50:14 +0000234 sh_chan->xmit_shift = calc_xmit_shift(sh_chan, val);
Kuninori Morimoto5899a722011-06-17 08:20:40 +0000235 chcr_write(sh_chan, val);
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000236
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000237 return 0;
238}
239
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000240static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val)
241{
Kuninori Morimotoc4e0dd72011-06-16 05:08:09 +0000242 struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000243 struct sh_dmae_pdata *pdata = shdev->pdata;
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200244 const struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->shdma_chan.id];
Magnus Damm26fc02a2011-05-24 10:31:12 +0000245 u16 __iomem *addr = shdev->dmars;
Kuninori Morimoto090b9182011-06-16 05:08:28 +0000246 unsigned int shift = chan_pdata->dmars_bit;
Guennadi Liakhovetskifc461852010-01-19 07:24:55 +0000247
248 if (dmae_is_busy(sh_chan))
249 return -EBUSY;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000250
Kuninori Morimoto260bf2c2011-06-17 08:21:05 +0000251 if (pdata->no_dmars)
252 return 0;
253
Magnus Damm26fc02a2011-05-24 10:31:12 +0000254 /* in the case of a missing DMARS resource use first memory window */
255 if (!addr)
256 addr = (u16 __iomem *)shdev->chan_reg;
257 addr += chan_pdata->dmars / sizeof(u16);
258
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000259 __raw_writew((__raw_readw(addr) & (0xff00 >> shift)) | (val << shift),
260 addr);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000261
262 return 0;
263}
264
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200265static void sh_dmae_start_xfer(struct shdma_chan *schan,
266 struct shdma_desc *sdesc)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000267{
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200268 struct sh_dmae_chan *sh_chan = container_of(schan, struct sh_dmae_chan,
269 shdma_chan);
270 struct sh_dmae_desc *sh_desc = container_of(sdesc,
271 struct sh_dmae_desc, shdma_desc);
272 dev_dbg(sh_chan->shdma_chan.dev, "Queue #%d to %d: %u@%x -> %x\n",
273 sdesc->async_tx.cookie, sh_chan->shdma_chan.id,
274 sh_desc->hw.tcr, sh_desc->hw.sar, sh_desc->hw.dar);
275 /* Get the ld start address from ld_queue */
276 dmae_set_reg(sh_chan, &sh_desc->hw);
277 dmae_start(sh_chan);
278}
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000279
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200280static bool sh_dmae_channel_busy(struct shdma_chan *schan)
281{
282 struct sh_dmae_chan *sh_chan = container_of(schan, struct sh_dmae_chan,
283 shdma_chan);
284 return dmae_is_busy(sh_chan);
285}
Guennadi Liakhovetski7a1cd9a2011-08-18 16:55:27 +0200286
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200287static void sh_dmae_setup_xfer(struct shdma_chan *schan,
Guennadi Liakhovetskic2cdb7e2012-07-05 12:29:41 +0200288 int slave_id)
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200289{
290 struct sh_dmae_chan *sh_chan = container_of(schan, struct sh_dmae_chan,
291 shdma_chan);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000292
Guennadi Liakhovetskic2cdb7e2012-07-05 12:29:41 +0200293 if (slave_id >= 0) {
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200294 const struct sh_dmae_slave_config *cfg =
Guennadi Liakhovetskiecf90fb2012-07-05 12:29:40 +0200295 sh_chan->config;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000296
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200297 dmae_set_dmars(sh_chan, cfg->mid_rid);
298 dmae_set_chcr(sh_chan, cfg->chcr);
Guennadi Liakhovetskic11b46c322012-01-04 15:34:17 +0100299 } else {
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200300 dmae_init(sh_chan);
Guennadi Liakhovetski7a1cd9a2011-08-18 16:55:27 +0200301 }
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000302}
303
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200304static const struct sh_dmae_slave_config *dmae_find_slave(
Guennadi Liakhovetskic2cdb7e2012-07-05 12:29:41 +0200305 struct sh_dmae_chan *sh_chan, int slave_id)
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000306{
Kuninori Morimotoc4e0dd72011-06-16 05:08:09 +0000307 struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000308 struct sh_dmae_pdata *pdata = shdev->pdata;
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200309 const struct sh_dmae_slave_config *cfg;
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000310 int i;
311
Guennadi Liakhovetski341f4dc2012-07-05 12:29:37 +0200312 if (slave_id >= SH_DMA_SLAVE_NUMBER)
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000313 return NULL;
314
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200315 for (i = 0, cfg = pdata->slave; i < pdata->slave_num; i++, cfg++)
Guennadi Liakhovetski341f4dc2012-07-05 12:29:37 +0200316 if (cfg->slave_id == slave_id)
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200317 return cfg;
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000318
319 return NULL;
320}
321
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200322static int sh_dmae_set_slave(struct shdma_chan *schan,
Guennadi Liakhovetski1ff8df42012-07-05 12:29:42 +0200323 int slave_id, bool try)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000324{
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200325 struct sh_dmae_chan *sh_chan = container_of(schan, struct sh_dmae_chan,
326 shdma_chan);
Guennadi Liakhovetskic2cdb7e2012-07-05 12:29:41 +0200327 const struct sh_dmae_slave_config *cfg = dmae_find_slave(sh_chan, slave_id);
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200328 if (!cfg)
329 return -ENODEV;
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000330
Guennadi Liakhovetski1ff8df42012-07-05 12:29:42 +0200331 if (!try)
332 sh_chan->config = cfg;
Linus Walleijc3635c72010-03-26 16:44:01 -0700333
334 return 0;
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000335}
336
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200337static void dmae_halt(struct sh_dmae_chan *sh_chan)
Guennadi Liakhovetski3542a112009-12-17 09:41:39 -0700338{
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200339 struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
340 u32 chcr = chcr_read(sh_chan);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000341
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200342 chcr &= ~(CHCR_DE | CHCR_TE | shdev->chcr_ie_bit);
343 chcr_write(sh_chan, chcr);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000344}
345
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200346static int sh_dmae_desc_setup(struct shdma_chan *schan,
347 struct shdma_desc *sdesc,
348 dma_addr_t src, dma_addr_t dst, size_t *len)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000349{
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200350 struct sh_dmae_desc *sh_desc = container_of(sdesc,
351 struct sh_dmae_desc, shdma_desc);
352
353 if (*len > schan->max_xfer_len)
354 *len = schan->max_xfer_len;
355
356 sh_desc->hw.sar = src;
357 sh_desc->hw.dar = dst;
358 sh_desc->hw.tcr = *len;
359
360 return 0;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000361}
362
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200363static void sh_dmae_halt(struct shdma_chan *schan)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000364{
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200365 struct sh_dmae_chan *sh_chan = container_of(schan, struct sh_dmae_chan,
366 shdma_chan);
367 dmae_halt(sh_chan);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000368}
369
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200370static bool sh_dmae_chan_irq(struct shdma_chan *schan, int irq)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000371{
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200372 struct sh_dmae_chan *sh_chan = container_of(schan, struct sh_dmae_chan,
373 shdma_chan);
Guennadi Liakhovetski7a1cd9a2011-08-18 16:55:27 +0200374
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200375 if (!(chcr_read(sh_chan) & CHCR_TE))
376 return false;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000377
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200378 /* DMA stop */
379 dmae_halt(sh_chan);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000380
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200381 return true;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000382}
383
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000384/* Called from error IRQ or NMI */
385static bool sh_dmae_reset(struct sh_dmae_device *shdev)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000386{
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200387 bool ret;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000388
Guennadi Liakhovetski47a4dc22010-02-11 16:50:05 +0000389 /* halt the dma controller */
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000390 sh_dmae_ctl_stop(shdev);
Guennadi Liakhovetski47a4dc22010-02-11 16:50:05 +0000391
392 /* We cannot detect, which channel caused the error, have to reset all */
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200393 ret = shdma_reset(&shdev->shdma_dev);
Paul Mundt03aa18f2010-12-17 19:16:10 +0900394
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000395 sh_dmae_rst(shdev);
Guennadi Liakhovetski47a4dc22010-02-11 16:50:05 +0000396
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200397 return ret;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000398}
Paul Mundt03aa18f2010-12-17 19:16:10 +0900399
400static irqreturn_t sh_dmae_err(int irq, void *data)
401{
Yoshihiro Shimodaff7690b2011-02-09 07:46:47 +0000402 struct sh_dmae_device *shdev = data;
403
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000404 if (!(dmaor_read(shdev) & DMAOR_AE))
Yoshihiro Shimodaff7690b2011-02-09 07:46:47 +0000405 return IRQ_NONE;
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000406
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200407 sh_dmae_reset(shdev);
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000408 return IRQ_HANDLED;
Paul Mundt03aa18f2010-12-17 19:16:10 +0900409}
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000410
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200411static bool sh_dmae_desc_completed(struct shdma_chan *schan,
412 struct shdma_desc *sdesc)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000413{
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200414 struct sh_dmae_chan *sh_chan = container_of(schan,
415 struct sh_dmae_chan, shdma_chan);
416 struct sh_dmae_desc *sh_desc = container_of(sdesc,
417 struct sh_dmae_desc, shdma_desc);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000418 u32 sar_buf = sh_dmae_readl(sh_chan, SAR);
Guennadi Liakhovetskicfefe992010-02-03 14:46:41 +0000419 u32 dar_buf = sh_dmae_readl(sh_chan, DAR);
Guennadi Liakhovetski86d61b32009-12-10 18:35:07 +0100420
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200421 return (sdesc->direction == DMA_DEV_TO_MEM &&
422 (sh_desc->hw.dar + sh_desc->hw.tcr) == dar_buf) ||
423 (sdesc->direction != DMA_DEV_TO_MEM &&
424 (sh_desc->hw.sar + sh_desc->hw.tcr) == sar_buf);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000425}
426
Paul Mundt03aa18f2010-12-17 19:16:10 +0900427static bool sh_dmae_nmi_notify(struct sh_dmae_device *shdev)
428{
Paul Mundt03aa18f2010-12-17 19:16:10 +0900429 /* Fast path out if NMIF is not asserted for this controller */
430 if ((dmaor_read(shdev) & DMAOR_NMIF) == 0)
431 return false;
432
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000433 return sh_dmae_reset(shdev);
Paul Mundt03aa18f2010-12-17 19:16:10 +0900434}
435
436static int sh_dmae_nmi_handler(struct notifier_block *self,
437 unsigned long cmd, void *data)
438{
439 struct sh_dmae_device *shdev;
440 int ret = NOTIFY_DONE;
441 bool triggered;
442
443 /*
444 * Only concern ourselves with NMI events.
445 *
446 * Normally we would check the die chain value, but as this needs
447 * to be architecture independent, check for NMI context instead.
448 */
449 if (!in_nmi())
450 return NOTIFY_DONE;
451
452 rcu_read_lock();
453 list_for_each_entry_rcu(shdev, &sh_dmae_devices, node) {
454 /*
455 * Only stop if one of the controllers has NMIF asserted,
456 * we do not want to interfere with regular address error
457 * handling or NMI events that don't concern the DMACs.
458 */
459 triggered = sh_dmae_nmi_notify(shdev);
460 if (triggered == true)
461 ret = NOTIFY_OK;
462 }
463 rcu_read_unlock();
464
465 return ret;
466}
467
468static struct notifier_block sh_dmae_nmi_notifier __read_mostly = {
469 .notifier_call = sh_dmae_nmi_handler,
470
471 /* Run before NMI debug handler and KGDB */
472 .priority = 1,
473};
474
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000475static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id,
476 int irq, unsigned long flags)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000477{
Guennadi Liakhovetski5bac9422010-04-21 15:36:49 +0000478 const struct sh_dmae_channel *chan_pdata = &shdev->pdata->channel[id];
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200479 struct shdma_dev *sdev = &shdev->shdma_dev;
480 struct platform_device *pdev = to_platform_device(sdev->dma_dev.dev);
481 struct sh_dmae_chan *sh_chan;
482 struct shdma_chan *schan;
483 int err;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000484
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200485 sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL);
486 if (!sh_chan) {
487 dev_err(sdev->dma_dev.dev,
Guennadi Liakhovetski86d61b32009-12-10 18:35:07 +0100488 "No free memory for allocating dma channels!\n");
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000489 return -ENOMEM;
490 }
491
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200492 schan = &sh_chan->shdma_chan;
493 schan->max_xfer_len = SH_DMA_TCR_MAX + 1;
Guennadi Liakhovetski7a1cd9a2011-08-18 16:55:27 +0200494
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200495 shdma_chan_probe(sdev, schan, id);
Guennadi Liakhovetski8b1935e2010-02-11 16:50:14 +0000496
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200497 sh_chan->base = shdev->chan_reg + chan_pdata->offset / sizeof(u32);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000498
499 /* set up channel irq */
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200500 if (pdev->id >= 0)
501 snprintf(sh_chan->dev_id, sizeof(sh_chan->dev_id),
502 "sh-dmae%d.%d", pdev->id, id);
503 else
504 snprintf(sh_chan->dev_id, sizeof(sh_chan->dev_id),
505 "sh-dma%d", id);
506
507 err = shdma_request_irq(schan, irq, flags, sh_chan->dev_id);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000508 if (err) {
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200509 dev_err(sdev->dma_dev.dev,
510 "DMA channel %d request_irq error %d\n",
511 id, err);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000512 goto err_no_irq;
513 }
514
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200515 shdev->chan[id] = sh_chan;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000516 return 0;
517
518err_no_irq:
519 /* remove from dmaengine device node */
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200520 shdma_chan_remove(schan);
521 kfree(sh_chan);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000522 return err;
523}
524
525static void sh_dmae_chan_remove(struct sh_dmae_device *shdev)
526{
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200527 struct dma_device *dma_dev = &shdev->shdma_dev.dma_dev;
528 struct shdma_chan *schan;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000529 int i;
530
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200531 shdma_for_each_chan(schan, &shdev->shdma_dev, i) {
532 struct sh_dmae_chan *sh_chan = container_of(schan,
533 struct sh_dmae_chan, shdma_chan);
534 BUG_ON(!schan);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000535
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200536 shdma_free_irq(&sh_chan->shdma_chan);
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000537
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200538 shdma_chan_remove(schan);
539 kfree(sh_chan);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000540 }
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200541 dma_dev->chancnt = 0;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000542}
543
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200544static void sh_dmae_shutdown(struct platform_device *pdev)
545{
546 struct sh_dmae_device *shdev = platform_get_drvdata(pdev);
547 sh_dmae_ctl_stop(shdev);
548}
549
550static int sh_dmae_runtime_suspend(struct device *dev)
551{
552 return 0;
553}
554
555static int sh_dmae_runtime_resume(struct device *dev)
556{
557 struct sh_dmae_device *shdev = dev_get_drvdata(dev);
558
559 return sh_dmae_rst(shdev);
560}
561
562#ifdef CONFIG_PM
563static int sh_dmae_suspend(struct device *dev)
564{
565 return 0;
566}
567
568static int sh_dmae_resume(struct device *dev)
569{
570 struct sh_dmae_device *shdev = dev_get_drvdata(dev);
571 int i, ret;
572
573 ret = sh_dmae_rst(shdev);
574 if (ret < 0)
575 dev_err(dev, "Failed to reset!\n");
576
577 for (i = 0; i < shdev->pdata->channel_num; i++) {
578 struct sh_dmae_chan *sh_chan = shdev->chan[i];
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200579
580 if (!sh_chan->shdma_chan.desc_num)
581 continue;
582
Guennadi Liakhovetskic2cdb7e2012-07-05 12:29:41 +0200583 if (sh_chan->shdma_chan.slave_id >= 0) {
Guennadi Liakhovetskiecf90fb2012-07-05 12:29:40 +0200584 const struct sh_dmae_slave_config *cfg = sh_chan->config;
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200585 dmae_set_dmars(sh_chan, cfg->mid_rid);
586 dmae_set_chcr(sh_chan, cfg->chcr);
587 } else {
588 dmae_init(sh_chan);
589 }
590 }
591
592 return 0;
593}
594#else
595#define sh_dmae_suspend NULL
596#define sh_dmae_resume NULL
597#endif
598
599const struct dev_pm_ops sh_dmae_pm = {
600 .suspend = sh_dmae_suspend,
601 .resume = sh_dmae_resume,
602 .runtime_suspend = sh_dmae_runtime_suspend,
603 .runtime_resume = sh_dmae_runtime_resume,
604};
605
606static dma_addr_t sh_dmae_slave_addr(struct shdma_chan *schan)
607{
Guennadi Liakhovetskiecf90fb2012-07-05 12:29:40 +0200608 struct sh_dmae_chan *sh_chan = container_of(schan,
609 struct sh_dmae_chan, shdma_chan);
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200610
611 /*
Guennadi Liakhovetskiecf90fb2012-07-05 12:29:40 +0200612 * Implicit BUG_ON(!sh_chan->config)
613 * This is an exclusive slave DMA operation, may only be called after a
614 * successful slave configuration.
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200615 */
Guennadi Liakhovetskiecf90fb2012-07-05 12:29:40 +0200616 return sh_chan->config->addr;
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200617}
618
619static struct shdma_desc *sh_dmae_embedded_desc(void *buf, int i)
620{
621 return &((struct sh_dmae_desc *)buf)[i].shdma_desc;
622}
623
624static const struct shdma_ops sh_dmae_shdma_ops = {
625 .desc_completed = sh_dmae_desc_completed,
626 .halt_channel = sh_dmae_halt,
627 .channel_busy = sh_dmae_channel_busy,
628 .slave_addr = sh_dmae_slave_addr,
629 .desc_setup = sh_dmae_desc_setup,
630 .set_slave = sh_dmae_set_slave,
631 .setup_xfer = sh_dmae_setup_xfer,
632 .start_xfer = sh_dmae_start_xfer,
633 .embedded_desc = sh_dmae_embedded_desc,
634 .chan_irq = sh_dmae_chan_irq,
635};
636
637static int __devinit sh_dmae_probe(struct platform_device *pdev)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000638{
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000639 struct sh_dmae_pdata *pdata = pdev->dev.platform_data;
640 unsigned long irqflags = IRQF_DISABLED,
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200641 chan_flag[SH_DMAE_MAX_CHANNELS] = {};
642 int errirq, chan_irq[SH_DMAE_MAX_CHANNELS];
Magnus Damm300e5f92011-05-24 10:31:20 +0000643 int err, i, irq_cnt = 0, irqres = 0, irq_cap = 0;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000644 struct sh_dmae_device *shdev;
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200645 struct dma_device *dma_dev;
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000646 struct resource *chan, *dmars, *errirq_res, *chanirq_res;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000647
Dan Williams56adf7e2009-11-22 12:10:10 -0700648 /* get platform data */
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000649 if (!pdata || !pdata->channel_num)
Dan Williams56adf7e2009-11-22 12:10:10 -0700650 return -ENODEV;
651
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000652 chan = platform_get_resource(pdev, IORESOURCE_MEM, 0);
Magnus Damm26fc02a2011-05-24 10:31:12 +0000653 /* DMARS area is optional */
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000654 dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1);
655 /*
656 * IRQ resources:
657 * 1. there always must be at least one IRQ IO-resource. On SH4 it is
658 * the error IRQ, in which case it is the only IRQ in this resource:
659 * start == end. If it is the only IRQ resource, all channels also
660 * use the same IRQ.
661 * 2. DMA channel IRQ resources can be specified one per resource or in
662 * ranges (start != end)
663 * 3. iff all events (channels and, optionally, error) on this
664 * controller use the same IRQ, only one IRQ resource can be
665 * specified, otherwise there must be one IRQ per channel, even if
666 * some of them are equal
667 * 4. if all IRQs on this controller are equal or if some specific IRQs
668 * specify IORESOURCE_IRQ_SHAREABLE in their resources, they will be
669 * requested with the IRQF_SHARED flag
670 */
671 errirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
672 if (!chan || !errirq_res)
673 return -ENODEV;
674
675 if (!request_mem_region(chan->start, resource_size(chan), pdev->name)) {
676 dev_err(&pdev->dev, "DMAC register region already claimed\n");
677 return -EBUSY;
678 }
679
680 if (dmars && !request_mem_region(dmars->start, resource_size(dmars), pdev->name)) {
681 dev_err(&pdev->dev, "DMAC DMARS region already claimed\n");
682 err = -EBUSY;
683 goto ermrdmars;
684 }
685
686 err = -ENOMEM;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000687 shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL);
688 if (!shdev) {
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000689 dev_err(&pdev->dev, "Not enough memory\n");
690 goto ealloc;
691 }
692
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200693 dma_dev = &shdev->shdma_dev.dma_dev;
694
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000695 shdev->chan_reg = ioremap(chan->start, resource_size(chan));
696 if (!shdev->chan_reg)
697 goto emapchan;
698 if (dmars) {
699 shdev->dmars = ioremap(dmars->start, resource_size(dmars));
700 if (!shdev->dmars)
701 goto emapdmars;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000702 }
703
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200704 if (!pdata->slave_only)
705 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
706 if (pdata->slave && pdata->slave_num)
707 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
708
709 /* Default transfer size of 32 bytes requires 32-byte alignment */
710 dma_dev->copy_align = LOG2_DEFAULT_XFER_SIZE;
711
712 shdev->shdma_dev.ops = &sh_dmae_shdma_ops;
713 shdev->shdma_dev.desc_size = sizeof(struct sh_dmae_desc);
714 err = shdma_init(&pdev->dev, &shdev->shdma_dev,
715 pdata->channel_num);
716 if (err < 0)
717 goto eshdma;
718
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000719 /* platform data */
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200720 shdev->pdata = pdev->dev.platform_data;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000721
Kuninori Morimoto5899a722011-06-17 08:20:40 +0000722 if (pdata->chcr_offset)
723 shdev->chcr_offset = pdata->chcr_offset;
724 else
725 shdev->chcr_offset = CHCR;
726
Kuninori Morimoto67c62692011-06-17 08:20:51 +0000727 if (pdata->chcr_ie_bit)
728 shdev->chcr_ie_bit = pdata->chcr_ie_bit;
729 else
730 shdev->chcr_ie_bit = CHCR_IE;
731
Paul Mundt5c2de442011-05-31 15:53:03 +0900732 platform_set_drvdata(pdev, shdev);
733
Guennadi Liakhovetski20f2a3b2010-02-11 16:50:18 +0000734 pm_runtime_enable(&pdev->dev);
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200735 err = pm_runtime_get_sync(&pdev->dev);
736 if (err < 0)
737 dev_err(&pdev->dev, "%s(): GET = %d\n", __func__, err);
Guennadi Liakhovetski20f2a3b2010-02-11 16:50:18 +0000738
Guennadi Liakhovetski31705e22011-05-02 07:59:02 +0000739 spin_lock_irq(&sh_dmae_lock);
Paul Mundt03aa18f2010-12-17 19:16:10 +0900740 list_add_tail_rcu(&shdev->node, &sh_dmae_devices);
Guennadi Liakhovetski31705e22011-05-02 07:59:02 +0000741 spin_unlock_irq(&sh_dmae_lock);
Paul Mundt03aa18f2010-12-17 19:16:10 +0900742
Guennadi Liakhovetski2dc66662011-04-29 17:09:21 +0000743 /* reset dma controller - only needed as a test */
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000744 err = sh_dmae_rst(shdev);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000745 if (err)
746 goto rst_err;
747
Magnus Damm927a7c92010-03-19 04:47:19 +0000748#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE)
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000749 chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 1);
750
751 if (!chanirq_res)
752 chanirq_res = errirq_res;
753 else
754 irqres++;
755
756 if (chanirq_res == errirq_res ||
757 (errirq_res->flags & IORESOURCE_BITS) == IORESOURCE_IRQ_SHAREABLE)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000758 irqflags = IRQF_SHARED;
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000759
760 errirq = errirq_res->start;
761
762 err = request_irq(errirq, sh_dmae_err, irqflags,
763 "DMAC Address Error", shdev);
764 if (err) {
765 dev_err(&pdev->dev,
766 "DMA failed requesting irq #%d, error %d\n",
767 errirq, err);
768 goto eirq_err;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000769 }
770
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000771#else
772 chanirq_res = errirq_res;
Magnus Damm927a7c92010-03-19 04:47:19 +0000773#endif /* CONFIG_CPU_SH4 || CONFIG_ARCH_SHMOBILE */
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000774
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000775 if (chanirq_res->start == chanirq_res->end &&
776 !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) {
777 /* Special case - all multiplexed */
778 for (; irq_cnt < pdata->channel_num; irq_cnt++) {
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200779 if (irq_cnt < SH_DMAE_MAX_CHANNELS) {
Magnus Damm300e5f92011-05-24 10:31:20 +0000780 chan_irq[irq_cnt] = chanirq_res->start;
781 chan_flag[irq_cnt] = IRQF_SHARED;
782 } else {
783 irq_cap = 1;
784 break;
785 }
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000786 }
787 } else {
788 do {
789 for (i = chanirq_res->start; i <= chanirq_res->end; i++) {
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200790 if (irq_cnt >= SH_DMAE_MAX_CHANNELS) {
Magnus Dammdcee0bb2011-06-09 06:35:08 +0000791 irq_cap = 1;
792 break;
793 }
794
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000795 if ((errirq_res->flags & IORESOURCE_BITS) ==
796 IORESOURCE_IRQ_SHAREABLE)
797 chan_flag[irq_cnt] = IRQF_SHARED;
798 else
799 chan_flag[irq_cnt] = IRQF_DISABLED;
800 dev_dbg(&pdev->dev,
801 "Found IRQ %d for channel %d\n",
802 i, irq_cnt);
803 chan_irq[irq_cnt++] = i;
Magnus Damm300e5f92011-05-24 10:31:20 +0000804 }
805
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200806 if (irq_cnt >= SH_DMAE_MAX_CHANNELS)
Magnus Damm300e5f92011-05-24 10:31:20 +0000807 break;
Magnus Dammdcee0bb2011-06-09 06:35:08 +0000808
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000809 chanirq_res = platform_get_resource(pdev,
810 IORESOURCE_IRQ, ++irqres);
811 } while (irq_cnt < pdata->channel_num && chanirq_res);
812 }
813
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000814 /* Create DMA Channel */
Magnus Damm300e5f92011-05-24 10:31:20 +0000815 for (i = 0; i < irq_cnt; i++) {
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000816 err = sh_dmae_chan_probe(shdev, i, chan_irq[i], chan_flag[i]);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000817 if (err)
818 goto chan_probe_err;
819 }
820
Magnus Damm300e5f92011-05-24 10:31:20 +0000821 if (irq_cap)
822 dev_notice(&pdev->dev, "Attempting to register %d DMA "
823 "channels when a maximum of %d are supported.\n",
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200824 pdata->channel_num, SH_DMAE_MAX_CHANNELS);
Magnus Damm300e5f92011-05-24 10:31:20 +0000825
Guennadi Liakhovetski20f2a3b2010-02-11 16:50:18 +0000826 pm_runtime_put(&pdev->dev);
827
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200828 err = dma_async_device_register(&shdev->shdma_dev.dma_dev);
829 if (err < 0)
830 goto edmadevreg;
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000831
832 return err;
833
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200834edmadevreg:
835 pm_runtime_get(&pdev->dev);
836
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000837chan_probe_err:
838 sh_dmae_chan_remove(shdev);
Magnus Damm300e5f92011-05-24 10:31:20 +0000839
Magnus Damm927a7c92010-03-19 04:47:19 +0000840#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE)
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000841 free_irq(errirq, shdev);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000842eirq_err:
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000843#endif
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000844rst_err:
Guennadi Liakhovetski31705e22011-05-02 07:59:02 +0000845 spin_lock_irq(&sh_dmae_lock);
Paul Mundt03aa18f2010-12-17 19:16:10 +0900846 list_del_rcu(&shdev->node);
Guennadi Liakhovetski31705e22011-05-02 07:59:02 +0000847 spin_unlock_irq(&sh_dmae_lock);
Paul Mundt03aa18f2010-12-17 19:16:10 +0900848
Guennadi Liakhovetski20f2a3b2010-02-11 16:50:18 +0000849 pm_runtime_put(&pdev->dev);
Guennadi Liakhovetski467017b2011-04-29 17:09:25 +0000850 pm_runtime_disable(&pdev->dev);
851
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200852 platform_set_drvdata(pdev, NULL);
853 shdma_cleanup(&shdev->shdma_dev);
854eshdma:
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000855 if (dmars)
856 iounmap(shdev->dmars);
857emapdmars:
858 iounmap(shdev->chan_reg);
Guennadi Liakhovetski31705e22011-05-02 07:59:02 +0000859 synchronize_rcu();
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000860emapchan:
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000861 kfree(shdev);
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000862ealloc:
863 if (dmars)
864 release_mem_region(dmars->start, resource_size(dmars));
865ermrdmars:
866 release_mem_region(chan->start, resource_size(chan));
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000867
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000868 return err;
869}
870
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200871static int __devexit sh_dmae_remove(struct platform_device *pdev)
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000872{
873 struct sh_dmae_device *shdev = platform_get_drvdata(pdev);
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200874 struct dma_device *dma_dev = &shdev->shdma_dev.dma_dev;
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000875 struct resource *res;
876 int errirq = platform_get_irq(pdev, 0);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000877
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200878 dma_async_device_unregister(dma_dev);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000879
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000880 if (errirq > 0)
881 free_irq(errirq, shdev);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000882
Guennadi Liakhovetski31705e22011-05-02 07:59:02 +0000883 spin_lock_irq(&sh_dmae_lock);
Paul Mundt03aa18f2010-12-17 19:16:10 +0900884 list_del_rcu(&shdev->node);
Guennadi Liakhovetski31705e22011-05-02 07:59:02 +0000885 spin_unlock_irq(&sh_dmae_lock);
Paul Mundt03aa18f2010-12-17 19:16:10 +0900886
Guennadi Liakhovetski20f2a3b2010-02-11 16:50:18 +0000887 pm_runtime_disable(&pdev->dev);
888
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200889 sh_dmae_chan_remove(shdev);
890 shdma_cleanup(&shdev->shdma_dev);
891
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000892 if (shdev->dmars)
893 iounmap(shdev->dmars);
894 iounmap(shdev->chan_reg);
895
Paul Mundt5c2de442011-05-31 15:53:03 +0900896 platform_set_drvdata(pdev, NULL);
897
Guennadi Liakhovetski31705e22011-05-02 07:59:02 +0000898 synchronize_rcu();
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000899 kfree(shdev);
900
Guennadi Liakhovetski027811b2010-02-11 16:50:10 +0000901 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
902 if (res)
903 release_mem_region(res->start, resource_size(res));
904 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
905 if (res)
906 release_mem_region(res->start, resource_size(res));
907
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000908 return 0;
909}
910
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000911static struct platform_driver sh_dmae_driver = {
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200912 .driver = {
Guennadi Liakhovetski7a5c1062010-05-21 15:28:51 +0000913 .owner = THIS_MODULE,
Guennadi Liakhovetski467017b2011-04-29 17:09:25 +0000914 .pm = &sh_dmae_pm,
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200915 .name = SH_DMAE_DRV_NAME,
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000916 },
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200917 .remove = __devexit_p(sh_dmae_remove),
918 .shutdown = sh_dmae_shutdown,
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000919};
920
921static int __init sh_dmae_init(void)
922{
Guennadi Liakhovetski661382f2011-01-06 17:04:50 +0000923 /* Wire up NMI handling */
924 int err = register_die_notifier(&sh_dmae_nmi_notifier);
925 if (err)
926 return err;
927
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000928 return platform_driver_probe(&sh_dmae_driver, sh_dmae_probe);
929}
930module_init(sh_dmae_init);
931
932static void __exit sh_dmae_exit(void)
933{
934 platform_driver_unregister(&sh_dmae_driver);
Guennadi Liakhovetski661382f2011-01-06 17:04:50 +0000935
936 unregister_die_notifier(&sh_dmae_nmi_notifier);
Nobuhiro Iwamatsud8902ad2009-09-07 03:26:23 +0000937}
938module_exit(sh_dmae_exit);
939
940MODULE_AUTHOR("Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>");
941MODULE_DESCRIPTION("Renesas SH DMA Engine driver");
942MODULE_LICENSE("GPL");
Guennadi Liakhovetskice3a1ab2012-05-09 17:09:21 +0200943MODULE_ALIAS("platform:" SH_DMAE_DRV_NAME);