arch/blackfin/kernel/bfin_dma_5xx.c - LeafOS-Devices/android_kernel_realme_mt6785 - Gitiles

 /*
  * File:         arch/blackfin/kernel/bfin_dma_5xx.c
  * Based on:
  * Author:
  *
  * Created:
  * Description:  This file contains the simple DMA Implementation for Blackfin
  *
  * Modified:
  *               Copyright 2004-2006 Analog Devices Inc.
  *
  * Bugs:         Enter bugs at http://blackfin.uclinux.org/
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, see the file COPYING, or write
  * to the Free Software Foundation, Inc.,
  * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */

 #include <linux/errno.h>
 #include <linux/module.h>
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/param.h>

 #include <asm/blackfin.h>
 #include <asm/dma.h>
 #include <asm/cacheflush.h>

 /**************************************************************************
  * Global Variables
 ***************************************************************************/

 static struct dma_channel dma_ch[MAX_BLACKFIN_DMA_CHANNEL];

 /*------------------------------------------------------------------------------
  *       Set the Buffer Clear bit in the Configuration register of specific DMA
  *       channel. This will stop the descriptor based DMA operation.
  *-----------------------------------------------------------------------------*/
 static void clear_dma_buffer(unsigned int channel)
 {
 	dma_ch[channel].regs->cfg |= RESTART;
 	SSYNC();
 	dma_ch[channel].regs->cfg &= ~RESTART;
 }

 static int __init blackfin_dma_init(void)
 {
 	int i;

 	printk(KERN_INFO "Blackfin DMA Controller\n");

 	for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++) {
 		dma_ch[i].chan_status = DMA_CHANNEL_FREE;
 		dma_ch[i].regs = dma_io_base_addr[i];
 		mutex_init(&(dma_ch[i].dmalock));
 	}
 	/* Mark MEMDMA Channel 0 as requested since we're using it internally */
 	dma_ch[CH_MEM_STREAM0_DEST].chan_status = DMA_CHANNEL_REQUESTED;
 	dma_ch[CH_MEM_STREAM0_SRC].chan_status = DMA_CHANNEL_REQUESTED;

 #if defined(CONFIG_DEB_DMA_URGENT)
 	bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
 			 | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
 #endif
 	return 0;
 }

 arch_initcall(blackfin_dma_init);

 /*------------------------------------------------------------------------------
  *	Request the specific DMA channel from the system.
  *-----------------------------------------------------------------------------*/
 int request_dma(unsigned int channel, char *device_id)
 {

 	pr_debug("request_dma() : BEGIN \n");

 #if defined(CONFIG_BF561) && ANOMALY_05000182
 	if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
 		if (get_cclk() > 500000000) {
 			printk(KERN_WARNING
 			       "Request IMDMA failed due to ANOMALY 05000182\n");
 			return -EFAULT;
 		}
 	}
 #endif

 	mutex_lock(&(dma_ch[channel].dmalock));

 	if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
 	    || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
 		mutex_unlock(&(dma_ch[channel].dmalock));
 		pr_debug("DMA CHANNEL IN USE  \n");
 		return -EBUSY;
 	} else {
 		dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
 		pr_debug("DMA CHANNEL IS ALLOCATED  \n");
 	}

 	mutex_unlock(&(dma_ch[channel].dmalock));

 #ifdef CONFIG_BF54x
 	if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
 		unsigned int per_map;
 		per_map = dma_ch[channel].regs->peripheral_map & 0xFFF;
 		if (strncmp(device_id, "BFIN_UART", 9) == 0)
 			dma_ch[channel].regs->peripheral_map = per_map |
 				((channel - CH_UART2_RX + 0xC)<<12);
 		else
 			dma_ch[channel].regs->peripheral_map = per_map |
 				((channel - CH_UART2_RX + 0x6)<<12);
 	}
 #endif

 	dma_ch[channel].device_id = device_id;
 	dma_ch[channel].irq_callback = NULL;

 	/* This is to be enabled by putting a restriction -
 	 * you have to request DMA, before doing any operations on
 	 * descriptor/channel
 	 */
 	pr_debug("request_dma() : END  \n");
 	return channel;
 }
 EXPORT_SYMBOL(request_dma);

 int set_dma_callback(unsigned int channel, dma_interrupt_t callback, void *data)
 {
 	BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 	       && channel < MAX_BLACKFIN_DMA_CHANNEL));

 	if (callback != NULL) {
 		int ret_val;
 		dma_ch[channel].irq = channel2irq(channel);
 		dma_ch[channel].data = data;

 		ret_val =
 		    request_irq(dma_ch[channel].irq, callback, IRQF_DISABLED,
 				dma_ch[channel].device_id, data);
 		if (ret_val) {
 			printk(KERN_NOTICE
 			       "Request irq in DMA engine failed.\n");
 			return -EPERM;
 		}
 		dma_ch[channel].irq_callback = callback;
 	}
 	return 0;
 }
 EXPORT_SYMBOL(set_dma_callback);

 void free_dma(unsigned int channel)
 {
 	pr_debug("freedma() : BEGIN \n");
 	BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 	       && channel < MAX_BLACKFIN_DMA_CHANNEL));

 	/* Halt the DMA */
 	disable_dma(channel);
 	clear_dma_buffer(channel);

 	if (dma_ch[channel].irq_callback != NULL)
 		free_irq(dma_ch[channel].irq, dma_ch[channel].data);

 	/* Clear the DMA Variable in the Channel */
 	mutex_lock(&(dma_ch[channel].dmalock));
 	dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
 	mutex_unlock(&(dma_ch[channel].dmalock));

 	pr_debug("freedma() : END \n");
 }
 EXPORT_SYMBOL(free_dma);

 void dma_enable_irq(unsigned int channel)
 {
 	pr_debug("dma_enable_irq() : BEGIN \n");
 	enable_irq(dma_ch[channel].irq);
 }
 EXPORT_SYMBOL(dma_enable_irq);

 void dma_disable_irq(unsigned int channel)
 {
 	pr_debug("dma_disable_irq() : BEGIN \n");
 	disable_irq(dma_ch[channel].irq);
 }
 EXPORT_SYMBOL(dma_disable_irq);

 int dma_channel_active(unsigned int channel)
 {
 	if (dma_ch[channel].chan_status == DMA_CHANNEL_FREE) {
 		return 0;
 	} else {
 		return 1;
 	}
 }
 EXPORT_SYMBOL(dma_channel_active);

 /*------------------------------------------------------------------------------
 *	stop the specific DMA channel.
 *-----------------------------------------------------------------------------*/
 void disable_dma(unsigned int channel)
 {
 	pr_debug("stop_dma() : BEGIN \n");
 	dma_ch[channel].regs->cfg &= ~DMAEN;	/* Clean the enable bit */
 	SSYNC();
 	dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
 	/* Needs to be enabled Later */
 	pr_debug("stop_dma() : END \n");
 	return;
 }
 EXPORT_SYMBOL(disable_dma);

 void enable_dma(unsigned int channel)
 {
 	pr_debug("enable_dma() : BEGIN \n");
 	dma_ch[channel].chan_status = DMA_CHANNEL_ENABLED;
 	dma_ch[channel].regs->curr_x_count = 0;
 	dma_ch[channel].regs->curr_y_count = 0;

 	dma_ch[channel].regs->cfg |= DMAEN;	/* Set the enable bit */
 	pr_debug("enable_dma() : END \n");
 	return;
 }
 EXPORT_SYMBOL(enable_dma);

 /*------------------------------------------------------------------------------
 *		Set the Start Address register for the specific DMA channel
 * 		This function can be used for register based DMA,
 *		to setup the start address
 *		addr:		Starting address of the DMA Data to be transferred.
 *-----------------------------------------------------------------------------*/
 void set_dma_start_addr(unsigned int channel, unsigned long addr)
 {
 	pr_debug("set_dma_start_addr() : BEGIN \n");
 	dma_ch[channel].regs->start_addr = addr;
 	pr_debug("set_dma_start_addr() : END\n");
 }
 EXPORT_SYMBOL(set_dma_start_addr);

 void set_dma_next_desc_addr(unsigned int channel, unsigned long addr)
 {
 	pr_debug("set_dma_next_desc_addr() : BEGIN \n");
 	dma_ch[channel].regs->next_desc_ptr = addr;
 	pr_debug("set_dma_next_desc_addr() : END\n");
 }
 EXPORT_SYMBOL(set_dma_next_desc_addr);

 void set_dma_curr_desc_addr(unsigned int channel, unsigned long addr)
 {
 	pr_debug("set_dma_curr_desc_addr() : BEGIN \n");
 	dma_ch[channel].regs->curr_desc_ptr = addr;
 	pr_debug("set_dma_curr_desc_addr() : END\n");
 }
 EXPORT_SYMBOL(set_dma_curr_desc_addr);

 void set_dma_x_count(unsigned int channel, unsigned short x_count)
 {
 	dma_ch[channel].regs->x_count = x_count;
 }
 EXPORT_SYMBOL(set_dma_x_count);

 void set_dma_y_count(unsigned int channel, unsigned short y_count)
 {
 	dma_ch[channel].regs->y_count = y_count;
 }
 EXPORT_SYMBOL(set_dma_y_count);

 void set_dma_x_modify(unsigned int channel, short x_modify)
 {
 	dma_ch[channel].regs->x_modify = x_modify;
 }
 EXPORT_SYMBOL(set_dma_x_modify);

 void set_dma_y_modify(unsigned int channel, short y_modify)
 {
 	dma_ch[channel].regs->y_modify = y_modify;
 }
 EXPORT_SYMBOL(set_dma_y_modify);

 void set_dma_config(unsigned int channel, unsigned short config)
 {
 	dma_ch[channel].regs->cfg = config;
 }
 EXPORT_SYMBOL(set_dma_config);

 unsigned short
 set_bfin_dma_config(char direction, char flow_mode,
 		    char intr_mode, char dma_mode, char width, char syncmode)
 {
 	unsigned short config;

 	config =
 	    ((direction << 1) | (width << 2) | (dma_mode << 4) |
 	     (intr_mode << 6) | (flow_mode << 12) | (syncmode << 5));
 	return config;
 }
 EXPORT_SYMBOL(set_bfin_dma_config);

 void set_dma_sg(unsigned int channel, struct dmasg *sg, int nr_sg)
 {
 	dma_ch[channel].regs->cfg |= ((nr_sg & 0x0F) << 8);
 	dma_ch[channel].regs->next_desc_ptr = (unsigned int)sg;
 }
 EXPORT_SYMBOL(set_dma_sg);

 void set_dma_curr_addr(unsigned int channel, unsigned long addr)
 {
 	dma_ch[channel].regs->curr_addr_ptr = addr;
 }
 EXPORT_SYMBOL(set_dma_curr_addr);

 /*------------------------------------------------------------------------------
  *	Get the DMA status of a specific DMA channel from the system.
  *-----------------------------------------------------------------------------*/
 unsigned short get_dma_curr_irqstat(unsigned int channel)
 {
 	return dma_ch[channel].regs->irq_status;
 }
 EXPORT_SYMBOL(get_dma_curr_irqstat);

 /*------------------------------------------------------------------------------
  *	Clear the DMA_DONE bit in DMA status. Stop the DMA completion interrupt.
  *-----------------------------------------------------------------------------*/
 void clear_dma_irqstat(unsigned int channel)
 {
 	dma_ch[channel].regs->irq_status |= 3;
 }
 EXPORT_SYMBOL(clear_dma_irqstat);

 /*------------------------------------------------------------------------------
  *	Get current DMA xcount of a specific DMA channel from the system.
  *-----------------------------------------------------------------------------*/
 unsigned short get_dma_curr_xcount(unsigned int channel)
 {
 	return dma_ch[channel].regs->curr_x_count;
 }
 EXPORT_SYMBOL(get_dma_curr_xcount);

 /*------------------------------------------------------------------------------
  *	Get current DMA ycount of a specific DMA channel from the system.
  *-----------------------------------------------------------------------------*/
 unsigned short get_dma_curr_ycount(unsigned int channel)
 {
 	return dma_ch[channel].regs->curr_y_count;
 }
 EXPORT_SYMBOL(get_dma_curr_ycount);

 unsigned long get_dma_next_desc_ptr(unsigned int channel)
 {
 	return dma_ch[channel].regs->next_desc_ptr;
 }
 EXPORT_SYMBOL(get_dma_next_desc_ptr);

 unsigned long get_dma_curr_desc_ptr(unsigned int channel)
 {
 	return dma_ch[channel].regs->curr_desc_ptr;
 }
 EXPORT_SYMBOL(get_dma_curr_desc_ptr);

 unsigned long get_dma_curr_addr(unsigned int channel)
 {
 	return dma_ch[channel].regs->curr_addr_ptr;
 }
 EXPORT_SYMBOL(get_dma_curr_addr);

 #ifdef CONFIG_PM
 int blackfin_dma_suspend(void)
 {
 	int i;

 #ifdef CONFIG_BF561	/* IMDMA channels doesn't have a PERIPHERAL_MAP */
 	for (i = 0; i <= CH_MEM_STREAM3_SRC; i++) {
 #else
 	for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++) {
 #endif
 		if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) {
 			printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
 			return -EBUSY;
 		}

 		dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
 	}

 	return 0;
 }

 void blackfin_dma_resume(void)
 {
 	int i;

 #ifdef CONFIG_BF561	/* IMDMA channels doesn't have a PERIPHERAL_MAP */
 	for (i = 0; i <= CH_MEM_STREAM3_SRC; i++)
 #else
 	for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++)
 #endif
 		dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
 }
 #endif

 static void *__dma_memcpy(void *dest, const void *src, size_t size)
 {
 	int direction;	/* 1 - address decrease, 0 - address increase */
 	int flag_align;	/* 1 - address aligned,  0 - address unaligned */
 	int flag_2D;	/* 1 - 2D DMA needed,	 0 - 1D DMA needed */
 	unsigned long flags;

 	if (size <= 0)
 		return NULL;

 	local_irq_save(flags);

 	if ((unsigned long)src < memory_end)
 		blackfin_dcache_flush_range((unsigned int)src,
 					    (unsigned int)(src + size));

 	if ((unsigned long)dest < memory_end)
 		blackfin_dcache_invalidate_range((unsigned int)dest,
 						 (unsigned int)(dest + size));

 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	if ((unsigned long)src < (unsigned long)dest)
 		direction = 1;
 	else
 		direction = 0;

 	if ((((unsigned long)dest % 2) == 0) && (((unsigned long)src % 2) == 0)
 	    && ((size % 2) == 0))
 		flag_align = 1;
 	else
 		flag_align = 0;

 	if (size > 0x10000)	/* size > 64K */
 		flag_2D = 1;
 	else
 		flag_2D = 0;

 	/* Setup destination and source start address */
 	if (direction) {
 		if (flag_align) {
 			bfin_write_MDMA_D0_START_ADDR(dest + size - 2);
 			bfin_write_MDMA_S0_START_ADDR(src + size - 2);
 		} else {
 			bfin_write_MDMA_D0_START_ADDR(dest + size - 1);
 			bfin_write_MDMA_S0_START_ADDR(src + size - 1);
 		}
 	} else {
 		bfin_write_MDMA_D0_START_ADDR(dest);
 		bfin_write_MDMA_S0_START_ADDR(src);
 	}

 	/* Setup destination and source xcount */
 	if (flag_2D) {
 		if (flag_align) {
 			bfin_write_MDMA_D0_X_COUNT(1024 / 2);
 			bfin_write_MDMA_S0_X_COUNT(1024 / 2);
 		} else {
 			bfin_write_MDMA_D0_X_COUNT(1024);
 			bfin_write_MDMA_S0_X_COUNT(1024);
 		}
 		bfin_write_MDMA_D0_Y_COUNT(size >> 10);
 		bfin_write_MDMA_S0_Y_COUNT(size >> 10);
 	} else {
 		if (flag_align) {
 			bfin_write_MDMA_D0_X_COUNT(size / 2);
 			bfin_write_MDMA_S0_X_COUNT(size / 2);
 		} else {
 			bfin_write_MDMA_D0_X_COUNT(size);
 			bfin_write_MDMA_S0_X_COUNT(size);
 		}
 	}

 	/* Setup destination and source xmodify and ymodify */
 	if (direction) {
 		if (flag_align) {
 			bfin_write_MDMA_D0_X_MODIFY(-2);
 			bfin_write_MDMA_S0_X_MODIFY(-2);
 			if (flag_2D) {
 				bfin_write_MDMA_D0_Y_MODIFY(-2);
 				bfin_write_MDMA_S0_Y_MODIFY(-2);
 			}
 		} else {
 			bfin_write_MDMA_D0_X_MODIFY(-1);
 			bfin_write_MDMA_S0_X_MODIFY(-1);
 			if (flag_2D) {
 				bfin_write_MDMA_D0_Y_MODIFY(-1);
 				bfin_write_MDMA_S0_Y_MODIFY(-1);
 			}
 		}
 	} else {
 		if (flag_align) {
 			bfin_write_MDMA_D0_X_MODIFY(2);
 			bfin_write_MDMA_S0_X_MODIFY(2);
 			if (flag_2D) {
 				bfin_write_MDMA_D0_Y_MODIFY(2);
 				bfin_write_MDMA_S0_Y_MODIFY(2);
 			}
 		} else {
 			bfin_write_MDMA_D0_X_MODIFY(1);
 			bfin_write_MDMA_S0_X_MODIFY(1);
 			if (flag_2D) {
 				bfin_write_MDMA_D0_Y_MODIFY(1);
 				bfin_write_MDMA_S0_Y_MODIFY(1);
 			}
 		}
 	}

 	/* Enable source DMA */
 	if (flag_2D) {
 		if (flag_align) {
 			bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D | WDSIZE_16);
 			bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D | WDSIZE_16);
 		} else {
 			bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D);
 			bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D);
 		}
 	} else {
 		if (flag_align) {
 			bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 			bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 		} else {
 			bfin_write_MDMA_S0_CONFIG(DMAEN);
 			bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN);
 		}
 	}

 	SSYNC();

 	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
 		;

 	bfin_write_MDMA_D0_IRQ_STATUS(bfin_read_MDMA_D0_IRQ_STATUS() |
 				      (DMA_DONE | DMA_ERR));

 	bfin_write_MDMA_S0_CONFIG(0);
 	bfin_write_MDMA_D0_CONFIG(0);

 	local_irq_restore(flags);

 	return dest;
 }

 void *dma_memcpy(void *dest, const void *src, size_t size)
 {
 	size_t bulk;
 	size_t rest;
 	void * addr;

 	bulk = (size >> 16) << 16;
 	rest = size - bulk;
 	if (bulk)
 		__dma_memcpy(dest, src, bulk);
 	addr = __dma_memcpy(dest+bulk, src+bulk, rest);
 	return addr;
 }
 EXPORT_SYMBOL(dma_memcpy);

 void *safe_dma_memcpy(void *dest, const void *src, size_t size)
 {
 	void *addr;
 	addr = dma_memcpy(dest, src, size);
 	return addr;
 }
 EXPORT_SYMBOL(safe_dma_memcpy);

 void dma_outsb(unsigned long addr, const void *buf, unsigned short len)
 {
 	unsigned long flags;

 	local_irq_save(flags);

 	blackfin_dcache_flush_range((unsigned int)buf,
 			 (unsigned int)(buf) + len);

 	bfin_write_MDMA_D0_START_ADDR(addr);
 	bfin_write_MDMA_D0_X_COUNT(len);
 	bfin_write_MDMA_D0_X_MODIFY(0);
 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_START_ADDR(buf);
 	bfin_write_MDMA_S0_X_COUNT(len);
 	bfin_write_MDMA_S0_X_MODIFY(1);
 	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
 	bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);

 	SSYNC();

 	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(0);
 	bfin_write_MDMA_D0_CONFIG(0);
 	local_irq_restore(flags);

 }
 EXPORT_SYMBOL(dma_outsb);


 void dma_insb(unsigned long addr, void *buf, unsigned short len)
 {
 	unsigned long flags;

 	blackfin_dcache_invalidate_range((unsigned int)buf,
 			 (unsigned int)(buf) + len);

 	local_irq_save(flags);
 	bfin_write_MDMA_D0_START_ADDR(buf);
 	bfin_write_MDMA_D0_X_COUNT(len);
 	bfin_write_MDMA_D0_X_MODIFY(1);
 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_START_ADDR(addr);
 	bfin_write_MDMA_S0_X_COUNT(len);
 	bfin_write_MDMA_S0_X_MODIFY(0);
 	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
 	bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);

 	SSYNC();

 	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(0);
 	bfin_write_MDMA_D0_CONFIG(0);
 	local_irq_restore(flags);

 }
 EXPORT_SYMBOL(dma_insb);

 void dma_outsw(unsigned long addr, const void  *buf, unsigned short len)
 {
 	unsigned long flags;

 	local_irq_save(flags);

 	blackfin_dcache_flush_range((unsigned int)buf,
 			 (unsigned int)(buf) + len * sizeof(short));

 	bfin_write_MDMA_D0_START_ADDR(addr);
 	bfin_write_MDMA_D0_X_COUNT(len);
 	bfin_write_MDMA_D0_X_MODIFY(0);
 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_START_ADDR(buf);
 	bfin_write_MDMA_S0_X_COUNT(len);
 	bfin_write_MDMA_S0_X_MODIFY(2);
 	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 	bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);

 	SSYNC();

 	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(0);
 	bfin_write_MDMA_D0_CONFIG(0);
 	local_irq_restore(flags);

 }
 EXPORT_SYMBOL(dma_outsw);

 void dma_insw(unsigned long addr, void *buf, unsigned short len)
 {
 	unsigned long flags;

 	blackfin_dcache_invalidate_range((unsigned int)buf,
 			 (unsigned int)(buf) + len * sizeof(short));

 	local_irq_save(flags);

 	bfin_write_MDMA_D0_START_ADDR(buf);
 	bfin_write_MDMA_D0_X_COUNT(len);
 	bfin_write_MDMA_D0_X_MODIFY(2);
 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_START_ADDR(addr);
 	bfin_write_MDMA_S0_X_COUNT(len);
 	bfin_write_MDMA_S0_X_MODIFY(0);
 	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 	bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);

 	SSYNC();

 	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(0);
 	bfin_write_MDMA_D0_CONFIG(0);
 	local_irq_restore(flags);

 }
 EXPORT_SYMBOL(dma_insw);

 void dma_outsl(unsigned long addr, const void *buf, unsigned short len)
 {
 	unsigned long flags;

 	local_irq_save(flags);

 	blackfin_dcache_flush_range((unsigned int)buf,
 			 (unsigned int)(buf) + len * sizeof(long));

 	bfin_write_MDMA_D0_START_ADDR(addr);
 	bfin_write_MDMA_D0_X_COUNT(len);
 	bfin_write_MDMA_D0_X_MODIFY(0);
 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_START_ADDR(buf);
 	bfin_write_MDMA_S0_X_COUNT(len);
 	bfin_write_MDMA_S0_X_MODIFY(4);
 	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
 	bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);

 	SSYNC();

 	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(0);
 	bfin_write_MDMA_D0_CONFIG(0);
 	local_irq_restore(flags);

 }
 EXPORT_SYMBOL(dma_outsl);

 void dma_insl(unsigned long addr, void *buf, unsigned short len)
 {
 	unsigned long flags;

 	blackfin_dcache_invalidate_range((unsigned int)buf,
 			 (unsigned int)(buf) + len * sizeof(long));

 	local_irq_save(flags);

 	bfin_write_MDMA_D0_START_ADDR(buf);
 	bfin_write_MDMA_D0_X_COUNT(len);
 	bfin_write_MDMA_D0_X_MODIFY(4);
 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_START_ADDR(addr);
 	bfin_write_MDMA_S0_X_COUNT(len);
 	bfin_write_MDMA_S0_X_MODIFY(0);
 	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
 	bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);

 	SSYNC();

 	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

 	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);

 	bfin_write_MDMA_S0_CONFIG(0);
 	bfin_write_MDMA_D0_CONFIG(0);
 	local_irq_restore(flags);

 }
 EXPORT_SYMBOL(dma_insl);
	/*
	* File: arch/blackfin/kernel/bfin_dma_5xx.c
	* Based on:
	* Author:
	*
	* Created:
	* Description: This file contains the simple DMA Implementation for Blackfin
	*
	* Modified:
	* Copyright 2004-2006 Analog Devices Inc.
	*
	* Bugs: Enter bugs at http://blackfin.uclinux.org/
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License as published by
	* the Free Software Foundation; either version 2 of the License, or
	* (at your option) any later version.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public License
	* along with this program; if not, see the file COPYING, or write
	* to the Free Software Foundation, Inc.,
	* 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <linux/errno.h>
	#include <linux/module.h>
	#include <linux/sched.h>
	#include <linux/interrupt.h>
	#include <linux/kernel.h>
	#include <linux/param.h>

	#include <asm/blackfin.h>
	#include <asm/dma.h>
	#include <asm/cacheflush.h>

	/**************************************************************************
	* Global Variables
	***************************************************************************/

	static struct dma_channel dma_ch[MAX_BLACKFIN_DMA_CHANNEL];

	/*------------------------------------------------------------------------------
	* Set the Buffer Clear bit in the Configuration register of specific DMA
	* channel. This will stop the descriptor based DMA operation.
	-----------------------------------------------------------------------------/
	static void clear_dma_buffer(unsigned int channel)
	{
	dma_ch[channel].regs->cfg \|= RESTART;
	SSYNC();
	dma_ch[channel].regs->cfg &= ~RESTART;
	}

	static int __init blackfin_dma_init(void)
	{
	int i;

	printk(KERN_INFO "Blackfin DMA Controller\n");

	for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++) {
	dma_ch[i].chan_status = DMA_CHANNEL_FREE;
	dma_ch[i].regs = dma_io_base_addr[i];
	mutex_init(&(dma_ch[i].dmalock));
	}
	/* Mark MEMDMA Channel 0 as requested since we're using it internally */
	dma_ch[CH_MEM_STREAM0_DEST].chan_status = DMA_CHANNEL_REQUESTED;
	dma_ch[CH_MEM_STREAM0_SRC].chan_status = DMA_CHANNEL_REQUESTED;

	#if defined(CONFIG_DEB_DMA_URGENT)
	bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
	\| DEB1_URGENT \| DEB2_URGENT \| DEB3_URGENT);
	#endif
	return 0;
	}

	arch_initcall(blackfin_dma_init);

	/*------------------------------------------------------------------------------
	* Request the specific DMA channel from the system.
	-----------------------------------------------------------------------------/
	int request_dma(unsigned int channel, char *device_id)
	{

	pr_debug("request_dma() : BEGIN \n");

	#if defined(CONFIG_BF561) && ANOMALY_05000182
	if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
	if (get_cclk() > 500000000) {
	printk(KERN_WARNING
	"Request IMDMA failed due to ANOMALY 05000182\n");
	return -EFAULT;
	}
	}
	#endif

	mutex_lock(&(dma_ch[channel].dmalock));

	if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
	\|\| (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
	mutex_unlock(&(dma_ch[channel].dmalock));
	pr_debug("DMA CHANNEL IN USE \n");
	return -EBUSY;
	} else {
	dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
	pr_debug("DMA CHANNEL IS ALLOCATED \n");
	}

	mutex_unlock(&(dma_ch[channel].dmalock));

	#ifdef CONFIG_BF54x
	if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
	unsigned int per_map;
	per_map = dma_ch[channel].regs->peripheral_map & 0xFFF;
	if (strncmp(device_id, "BFIN_UART", 9) == 0)
	dma_ch[channel].regs->peripheral_map = per_map \|
	((channel - CH_UART2_RX + 0xC)<<12);
	else
	dma_ch[channel].regs->peripheral_map = per_map \|
	((channel - CH_UART2_RX + 0x6)<<12);
	}
	#endif

	dma_ch[channel].device_id = device_id;
	dma_ch[channel].irq_callback = NULL;

	/* This is to be enabled by putting a restriction -
	* you have to request DMA, before doing any operations on
	* descriptor/channel
	*/
	pr_debug("request_dma() : END \n");
	return channel;
	}
	EXPORT_SYMBOL(request_dma);

	int set_dma_callback(unsigned int channel, dma_interrupt_t callback, void *data)
	{
	BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
	&& channel < MAX_BLACKFIN_DMA_CHANNEL));

	if (callback != NULL) {
	int ret_val;
	dma_ch[channel].irq = channel2irq(channel);
	dma_ch[channel].data = data;

	ret_val =
	request_irq(dma_ch[channel].irq, callback, IRQF_DISABLED,
	dma_ch[channel].device_id, data);
	if (ret_val) {
	printk(KERN_NOTICE
	"Request irq in DMA engine failed.\n");
	return -EPERM;
	}
	dma_ch[channel].irq_callback = callback;
	}
	return 0;
	}
	EXPORT_SYMBOL(set_dma_callback);

	void free_dma(unsigned int channel)
	{
	pr_debug("freedma() : BEGIN \n");
	BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
	&& channel < MAX_BLACKFIN_DMA_CHANNEL));

	/* Halt the DMA */
	disable_dma(channel);
	clear_dma_buffer(channel);

	if (dma_ch[channel].irq_callback != NULL)
	free_irq(dma_ch[channel].irq, dma_ch[channel].data);

	/* Clear the DMA Variable in the Channel */
	mutex_lock(&(dma_ch[channel].dmalock));
	dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
	mutex_unlock(&(dma_ch[channel].dmalock));

	pr_debug("freedma() : END \n");
	}
	EXPORT_SYMBOL(free_dma);

	void dma_enable_irq(unsigned int channel)
	{
	pr_debug("dma_enable_irq() : BEGIN \n");
	enable_irq(dma_ch[channel].irq);
	}
	EXPORT_SYMBOL(dma_enable_irq);

	void dma_disable_irq(unsigned int channel)
	{
	pr_debug("dma_disable_irq() : BEGIN \n");
	disable_irq(dma_ch[channel].irq);
	}
	EXPORT_SYMBOL(dma_disable_irq);

	int dma_channel_active(unsigned int channel)
	{
	if (dma_ch[channel].chan_status == DMA_CHANNEL_FREE) {
	return 0;
	} else {
	return 1;
	}
	}
	EXPORT_SYMBOL(dma_channel_active);

	/*------------------------------------------------------------------------------
	* stop the specific DMA channel.
	-----------------------------------------------------------------------------/
	void disable_dma(unsigned int channel)
	{
	pr_debug("stop_dma() : BEGIN \n");
	dma_ch[channel].regs->cfg &= ~DMAEN; /* Clean the enable bit */
	SSYNC();
	dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
	/* Needs to be enabled Later */
	pr_debug("stop_dma() : END \n");
	return;
	}
	EXPORT_SYMBOL(disable_dma);

	void enable_dma(unsigned int channel)
	{
	pr_debug("enable_dma() : BEGIN \n");
	dma_ch[channel].chan_status = DMA_CHANNEL_ENABLED;
	dma_ch[channel].regs->curr_x_count = 0;
	dma_ch[channel].regs->curr_y_count = 0;

	dma_ch[channel].regs->cfg \|= DMAEN; /* Set the enable bit */
	pr_debug("enable_dma() : END \n");
	return;
	}
	EXPORT_SYMBOL(enable_dma);

	/*------------------------------------------------------------------------------
	* Set the Start Address register for the specific DMA channel
	* This function can be used for register based DMA,
	* to setup the start address
	* addr: Starting address of the DMA Data to be transferred.
	-----------------------------------------------------------------------------/
	void set_dma_start_addr(unsigned int channel, unsigned long addr)
	{
	pr_debug("set_dma_start_addr() : BEGIN \n");
	dma_ch[channel].regs->start_addr = addr;
	pr_debug("set_dma_start_addr() : END\n");
	}
	EXPORT_SYMBOL(set_dma_start_addr);

	void set_dma_next_desc_addr(unsigned int channel, unsigned long addr)
	{
	pr_debug("set_dma_next_desc_addr() : BEGIN \n");
	dma_ch[channel].regs->next_desc_ptr = addr;
	pr_debug("set_dma_next_desc_addr() : END\n");
	}
	EXPORT_SYMBOL(set_dma_next_desc_addr);

	void set_dma_curr_desc_addr(unsigned int channel, unsigned long addr)
	{
	pr_debug("set_dma_curr_desc_addr() : BEGIN \n");
	dma_ch[channel].regs->curr_desc_ptr = addr;
	pr_debug("set_dma_curr_desc_addr() : END\n");
	}
	EXPORT_SYMBOL(set_dma_curr_desc_addr);

	void set_dma_x_count(unsigned int channel, unsigned short x_count)
	{
	dma_ch[channel].regs->x_count = x_count;
	}
	EXPORT_SYMBOL(set_dma_x_count);

	void set_dma_y_count(unsigned int channel, unsigned short y_count)
	{
	dma_ch[channel].regs->y_count = y_count;
	}
	EXPORT_SYMBOL(set_dma_y_count);

	void set_dma_x_modify(unsigned int channel, short x_modify)
	{
	dma_ch[channel].regs->x_modify = x_modify;
	}
	EXPORT_SYMBOL(set_dma_x_modify);

	void set_dma_y_modify(unsigned int channel, short y_modify)
	{
	dma_ch[channel].regs->y_modify = y_modify;
	}
	EXPORT_SYMBOL(set_dma_y_modify);

	void set_dma_config(unsigned int channel, unsigned short config)
	{
	dma_ch[channel].regs->cfg = config;
	}
	EXPORT_SYMBOL(set_dma_config);

	unsigned short
	set_bfin_dma_config(char direction, char flow_mode,
	char intr_mode, char dma_mode, char width, char syncmode)
	{
	unsigned short config;

	config =
	((direction << 1) \| (width << 2) \| (dma_mode << 4) \|
	(intr_mode << 6) \| (flow_mode << 12) \| (syncmode << 5));
	return config;
	}
	EXPORT_SYMBOL(set_bfin_dma_config);

	void set_dma_sg(unsigned int channel, struct dmasg *sg, int nr_sg)
	{
	dma_ch[channel].regs->cfg \|= ((nr_sg & 0x0F) << 8);
	dma_ch[channel].regs->next_desc_ptr = (unsigned int)sg;
	}
	EXPORT_SYMBOL(set_dma_sg);

	void set_dma_curr_addr(unsigned int channel, unsigned long addr)
	{
	dma_ch[channel].regs->curr_addr_ptr = addr;
	}
	EXPORT_SYMBOL(set_dma_curr_addr);

	/*------------------------------------------------------------------------------
	* Get the DMA status of a specific DMA channel from the system.
	-----------------------------------------------------------------------------/
	unsigned short get_dma_curr_irqstat(unsigned int channel)
	{
	return dma_ch[channel].regs->irq_status;
	}
	EXPORT_SYMBOL(get_dma_curr_irqstat);

	/*------------------------------------------------------------------------------
	* Clear the DMA_DONE bit in DMA status. Stop the DMA completion interrupt.
	-----------------------------------------------------------------------------/
	void clear_dma_irqstat(unsigned int channel)
	{
	dma_ch[channel].regs->irq_status \|= 3;
	}
	EXPORT_SYMBOL(clear_dma_irqstat);

	/*------------------------------------------------------------------------------
	* Get current DMA xcount of a specific DMA channel from the system.
	-----------------------------------------------------------------------------/
	unsigned short get_dma_curr_xcount(unsigned int channel)
	{
	return dma_ch[channel].regs->curr_x_count;
	}
	EXPORT_SYMBOL(get_dma_curr_xcount);

	/*------------------------------------------------------------------------------
	* Get current DMA ycount of a specific DMA channel from the system.
	-----------------------------------------------------------------------------/
	unsigned short get_dma_curr_ycount(unsigned int channel)
	{
	return dma_ch[channel].regs->curr_y_count;
	}
	EXPORT_SYMBOL(get_dma_curr_ycount);

	unsigned long get_dma_next_desc_ptr(unsigned int channel)
	{
	return dma_ch[channel].regs->next_desc_ptr;
	}
	EXPORT_SYMBOL(get_dma_next_desc_ptr);

	unsigned long get_dma_curr_desc_ptr(unsigned int channel)
	{
	return dma_ch[channel].regs->curr_desc_ptr;
	}
	EXPORT_SYMBOL(get_dma_curr_desc_ptr);

	unsigned long get_dma_curr_addr(unsigned int channel)
	{
	return dma_ch[channel].regs->curr_addr_ptr;
	}
	EXPORT_SYMBOL(get_dma_curr_addr);

	#ifdef CONFIG_PM
	int blackfin_dma_suspend(void)
	{
	int i;

	#ifdef CONFIG_BF561 /* IMDMA channels doesn't have a PERIPHERAL_MAP */
	for (i = 0; i <= CH_MEM_STREAM3_SRC; i++) {
	#else
	for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++) {
	#endif
	if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) {
	printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
	return -EBUSY;
	}

	dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
	}

	return 0;
	}

	void blackfin_dma_resume(void)
	{
	int i;

	#ifdef CONFIG_BF561 /* IMDMA channels doesn't have a PERIPHERAL_MAP */
	for (i = 0; i <= CH_MEM_STREAM3_SRC; i++)
	#else
	for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++)
	#endif
	dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
	}
	#endif

	static void __dma_memcpy(void dest, const void *src, size_t size)
	{
	int direction; /* 1 - address decrease, 0 - address increase */
	int flag_align; /* 1 - address aligned, 0 - address unaligned */
	int flag_2D; /* 1 - 2D DMA needed, 0 - 1D DMA needed */
	unsigned long flags;

	if (size <= 0)
	return NULL;

	local_irq_save(flags);

	if ((unsigned long)src < memory_end)
	blackfin_dcache_flush_range((unsigned int)src,
	(unsigned int)(src + size));

	if ((unsigned long)dest < memory_end)
	blackfin_dcache_invalidate_range((unsigned int)dest,
	(unsigned int)(dest + size));

	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	if ((unsigned long)src < (unsigned long)dest)
	direction = 1;
	else
	direction = 0;

	if ((((unsigned long)dest % 2) == 0) && (((unsigned long)src % 2) == 0)
	&& ((size % 2) == 0))
	flag_align = 1;
	else
	flag_align = 0;

	if (size > 0x10000) /* size > 64K */
	flag_2D = 1;
	else
	flag_2D = 0;

	/* Setup destination and source start address */
	if (direction) {
	if (flag_align) {
	bfin_write_MDMA_D0_START_ADDR(dest + size - 2);
	bfin_write_MDMA_S0_START_ADDR(src + size - 2);
	} else {
	bfin_write_MDMA_D0_START_ADDR(dest + size - 1);
	bfin_write_MDMA_S0_START_ADDR(src + size - 1);
	}
	} else {
	bfin_write_MDMA_D0_START_ADDR(dest);
	bfin_write_MDMA_S0_START_ADDR(src);
	}

	/* Setup destination and source xcount */
	if (flag_2D) {
	if (flag_align) {
	bfin_write_MDMA_D0_X_COUNT(1024 / 2);
	bfin_write_MDMA_S0_X_COUNT(1024 / 2);
	} else {
	bfin_write_MDMA_D0_X_COUNT(1024);
	bfin_write_MDMA_S0_X_COUNT(1024);
	}
	bfin_write_MDMA_D0_Y_COUNT(size >> 10);
	bfin_write_MDMA_S0_Y_COUNT(size >> 10);
	} else {
	if (flag_align) {
	bfin_write_MDMA_D0_X_COUNT(size / 2);
	bfin_write_MDMA_S0_X_COUNT(size / 2);
	} else {
	bfin_write_MDMA_D0_X_COUNT(size);
	bfin_write_MDMA_S0_X_COUNT(size);
	}
	}

	/* Setup destination and source xmodify and ymodify */
	if (direction) {
	if (flag_align) {
	bfin_write_MDMA_D0_X_MODIFY(-2);
	bfin_write_MDMA_S0_X_MODIFY(-2);
	if (flag_2D) {
	bfin_write_MDMA_D0_Y_MODIFY(-2);
	bfin_write_MDMA_S0_Y_MODIFY(-2);
	}
	} else {
	bfin_write_MDMA_D0_X_MODIFY(-1);
	bfin_write_MDMA_S0_X_MODIFY(-1);
	if (flag_2D) {
	bfin_write_MDMA_D0_Y_MODIFY(-1);
	bfin_write_MDMA_S0_Y_MODIFY(-1);
	}
	}
	} else {
	if (flag_align) {
	bfin_write_MDMA_D0_X_MODIFY(2);
	bfin_write_MDMA_S0_X_MODIFY(2);
	if (flag_2D) {
	bfin_write_MDMA_D0_Y_MODIFY(2);
	bfin_write_MDMA_S0_Y_MODIFY(2);
	}
	} else {
	bfin_write_MDMA_D0_X_MODIFY(1);
	bfin_write_MDMA_S0_X_MODIFY(1);
	if (flag_2D) {
	bfin_write_MDMA_D0_Y_MODIFY(1);
	bfin_write_MDMA_S0_Y_MODIFY(1);
	}
	}
	}

	/* Enable source DMA */
	if (flag_2D) {
	if (flag_align) {
	bfin_write_MDMA_S0_CONFIG(DMAEN \| DMA2D \| WDSIZE_16);
	bfin_write_MDMA_D0_CONFIG(WNR \| DI_EN \| DMAEN \| DMA2D \| WDSIZE_16);
	} else {
	bfin_write_MDMA_S0_CONFIG(DMAEN \| DMA2D);
	bfin_write_MDMA_D0_CONFIG(WNR \| DI_EN \| DMAEN \| DMA2D);
	}
	} else {
	if (flag_align) {
	bfin_write_MDMA_S0_CONFIG(DMAEN \| WDSIZE_16);
	bfin_write_MDMA_D0_CONFIG(WNR \| DI_EN \| DMAEN \| WDSIZE_16);
	} else {
	bfin_write_MDMA_S0_CONFIG(DMAEN);
	bfin_write_MDMA_D0_CONFIG(WNR \| DI_EN \| DMAEN);
	}
	}

	SSYNC();

	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
	;

	bfin_write_MDMA_D0_IRQ_STATUS(bfin_read_MDMA_D0_IRQ_STATUS() \|
	(DMA_DONE \| DMA_ERR));

	bfin_write_MDMA_S0_CONFIG(0);
	bfin_write_MDMA_D0_CONFIG(0);

	local_irq_restore(flags);

	return dest;
	}

	void dma_memcpy(void dest, const void *src, size_t size)
	{
	size_t bulk;
	size_t rest;
	void * addr;

	bulk = (size >> 16) << 16;
	rest = size - bulk;
	if (bulk)
	__dma_memcpy(dest, src, bulk);
	addr = __dma_memcpy(dest+bulk, src+bulk, rest);
	return addr;
	}
	EXPORT_SYMBOL(dma_memcpy);

	void safe_dma_memcpy(void dest, const void *src, size_t size)
	{
	void *addr;
	addr = dma_memcpy(dest, src, size);
	return addr;
	}
	EXPORT_SYMBOL(safe_dma_memcpy);

	void dma_outsb(unsigned long addr, const void *buf, unsigned short len)
	{
	unsigned long flags;

	local_irq_save(flags);

	blackfin_dcache_flush_range((unsigned int)buf,
	(unsigned int)(buf) + len);

	bfin_write_MDMA_D0_START_ADDR(addr);
	bfin_write_MDMA_D0_X_COUNT(len);
	bfin_write_MDMA_D0_X_MODIFY(0);
	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_START_ADDR(buf);
	bfin_write_MDMA_S0_X_COUNT(len);
	bfin_write_MDMA_S0_X_MODIFY(1);
	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(DMAEN \| WDSIZE_8);
	bfin_write_MDMA_D0_CONFIG(WNR \| DI_EN \| DMAEN \| WDSIZE_8);

	SSYNC();

	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(0);
	bfin_write_MDMA_D0_CONFIG(0);
	local_irq_restore(flags);

	}
	EXPORT_SYMBOL(dma_outsb);


	void dma_insb(unsigned long addr, void *buf, unsigned short len)
	{
	unsigned long flags;

	blackfin_dcache_invalidate_range((unsigned int)buf,
	(unsigned int)(buf) + len);

	local_irq_save(flags);
	bfin_write_MDMA_D0_START_ADDR(buf);
	bfin_write_MDMA_D0_X_COUNT(len);
	bfin_write_MDMA_D0_X_MODIFY(1);
	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_START_ADDR(addr);
	bfin_write_MDMA_S0_X_COUNT(len);
	bfin_write_MDMA_S0_X_MODIFY(0);
	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(DMAEN \| WDSIZE_8);
	bfin_write_MDMA_D0_CONFIG(WNR \| DI_EN \| DMAEN \| WDSIZE_8);

	SSYNC();

	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(0);
	bfin_write_MDMA_D0_CONFIG(0);
	local_irq_restore(flags);

	}
	EXPORT_SYMBOL(dma_insb);

	void dma_outsw(unsigned long addr, const void *buf, unsigned short len)
	{
	unsigned long flags;

	local_irq_save(flags);

	blackfin_dcache_flush_range((unsigned int)buf,
	(unsigned int)(buf) + len * sizeof(short));

	bfin_write_MDMA_D0_START_ADDR(addr);
	bfin_write_MDMA_D0_X_COUNT(len);
	bfin_write_MDMA_D0_X_MODIFY(0);
	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_START_ADDR(buf);
	bfin_write_MDMA_S0_X_COUNT(len);
	bfin_write_MDMA_S0_X_MODIFY(2);
	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(DMAEN \| WDSIZE_16);
	bfin_write_MDMA_D0_CONFIG(WNR \| DI_EN \| DMAEN \| WDSIZE_16);

	SSYNC();

	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(0);
	bfin_write_MDMA_D0_CONFIG(0);
	local_irq_restore(flags);

	}
	EXPORT_SYMBOL(dma_outsw);

	void dma_insw(unsigned long addr, void *buf, unsigned short len)
	{
	unsigned long flags;

	blackfin_dcache_invalidate_range((unsigned int)buf,
	(unsigned int)(buf) + len * sizeof(short));

	local_irq_save(flags);

	bfin_write_MDMA_D0_START_ADDR(buf);
	bfin_write_MDMA_D0_X_COUNT(len);
	bfin_write_MDMA_D0_X_MODIFY(2);
	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_START_ADDR(addr);
	bfin_write_MDMA_S0_X_COUNT(len);
	bfin_write_MDMA_S0_X_MODIFY(0);
	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(DMAEN \| WDSIZE_16);
	bfin_write_MDMA_D0_CONFIG(WNR \| DI_EN \| DMAEN \| WDSIZE_16);

	SSYNC();

	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(0);
	bfin_write_MDMA_D0_CONFIG(0);
	local_irq_restore(flags);

	}
	EXPORT_SYMBOL(dma_insw);

	void dma_outsl(unsigned long addr, const void *buf, unsigned short len)
	{
	unsigned long flags;

	local_irq_save(flags);

	blackfin_dcache_flush_range((unsigned int)buf,
	(unsigned int)(buf) + len * sizeof(long));

	bfin_write_MDMA_D0_START_ADDR(addr);
	bfin_write_MDMA_D0_X_COUNT(len);
	bfin_write_MDMA_D0_X_MODIFY(0);
	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_START_ADDR(buf);
	bfin_write_MDMA_S0_X_COUNT(len);
	bfin_write_MDMA_S0_X_MODIFY(4);
	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(DMAEN \| WDSIZE_32);
	bfin_write_MDMA_D0_CONFIG(WNR \| DI_EN \| DMAEN \| WDSIZE_32);

	SSYNC();

	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(0);
	bfin_write_MDMA_D0_CONFIG(0);
	local_irq_restore(flags);

	}
	EXPORT_SYMBOL(dma_outsl);

	void dma_insl(unsigned long addr, void *buf, unsigned short len)
	{
	unsigned long flags;

	blackfin_dcache_invalidate_range((unsigned int)buf,
	(unsigned int)(buf) + len * sizeof(long));

	local_irq_save(flags);

	bfin_write_MDMA_D0_START_ADDR(buf);
	bfin_write_MDMA_D0_X_COUNT(len);
	bfin_write_MDMA_D0_X_MODIFY(4);
	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_START_ADDR(addr);
	bfin_write_MDMA_S0_X_COUNT(len);
	bfin_write_MDMA_S0_X_MODIFY(0);
	bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(DMAEN \| WDSIZE_32);
	bfin_write_MDMA_D0_CONFIG(WNR \| DI_EN \| DMAEN \| WDSIZE_32);

	SSYNC();

	while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));

	bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE \| DMA_ERR);

	bfin_write_MDMA_S0_CONFIG(0);
	bfin_write_MDMA_D0_CONFIG(0);
	local_irq_restore(flags);

	}
	EXPORT_SYMBOL(dma_insl);