| /* |
| * CPPI 4.1 support |
| * |
| * Copyright (C) 2008-2009 MontaVista Software, Inc. <source@mvista.com> |
| * |
| * Based on the PAL CPPI 4.1 implementation |
| * Copyright (C) 1998-2006 Texas Instruments Incorporated |
| * |
| * This file contains the main implementation for CPPI 4.1 common peripherals, |
| * including the DMA Controllers and the Queue Managers. |
| * |
| * This program is free software; you can distribute it and/or modify it |
| * under the terms of the GNU General Public License (Version 2) as |
| * published by the Free Software Foundation. |
| * |
| * This program is distributed in the hope it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * for more details. |
| * |
| * You should have received a copy of the GNU General Public License along |
| * with this program; if not, write to the Free Software Foundation, Inc., |
| * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. |
| * |
| */ |
| |
| #include <linux/io.h> |
| #include <linux/init.h> |
| #include <linux/slab.h> |
| #include <linux/module.h> |
| #include <linux/dma-mapping.h> |
| |
| #include "cppi41.h" |
| |
| #undef CPPI41_DEBUG |
| |
| #ifdef CPPI41_DEBUG |
| #define DBG(format, args...) printk(format, ##args) |
| #else |
| #define DBG(format, args...) |
| #endif |
| |
| static struct { |
| void *virt_addr; |
| dma_addr_t phys_addr; |
| u32 size; |
| } linking_ram[CPPI41_NUM_QUEUE_MGR]; |
| |
| static u32 *allocated_queues[CPPI41_NUM_QUEUE_MGR]; |
| |
| /* First 32 packet descriptors are reserved for unallocated memory regions. */ |
| static u32 next_desc_index[CPPI41_NUM_QUEUE_MGR] = { 1 << 5 }; |
| static u8 next_mem_rgn[CPPI41_NUM_QUEUE_MGR]; |
| |
| static struct { |
| size_t rgn_size; |
| void *virt_addr; |
| dma_addr_t phys_addr; |
| struct cppi41_queue_obj queue_obj; |
| u8 mem_rgn; |
| u16 q_mgr; |
| u16 q_num; |
| } dma_teardown[CPPI41_NUM_DMA_BLOCK]; |
| |
| struct cppi41_dma_sched_tbl_t { |
| u8 pos; |
| u8 dma_ch; |
| u8 is_tx; |
| u8 enb; |
| }; |
| |
| struct cppi41_dma_sched_tbl_t dma_sched_tbl[MAX_SCHED_TBL_ENTRY] = { |
| /*pos dma_ch# is_tx enb/dis*/ |
| { 0, 0, 0, 1}, |
| { 1, 0, 1, 1}, |
| { 2, 1, 0, 1}, |
| { 3, 1, 1, 1}, |
| { 4, 2, 0, 1}, |
| { 5, 2, 1, 1}, |
| { 6, 3, 0, 1}, |
| { 7, 3, 1, 1} |
| }; |
| |
| struct cppi41_queue_mgr cppi41_queue_mgr[CPPI41_NUM_QUEUE_MGR]; |
| EXPORT_SYMBOL(cppi41_queue_mgr); |
| |
| struct cppi41_dma_block cppi41_dma_block[CPPI41_NUM_DMA_BLOCK]; |
| EXPORT_SYMBOL(cppi41_dma_block); |
| /******************** CPPI 4.1 Functions (External Interface) *****************/ |
| |
| int cppi41_queue_mgr_init(u8 q_mgr, dma_addr_t rgn0_base, u16 rgn0_size) |
| { |
| void __iomem *q_mgr_regs; |
| void *ptr; |
| |
| if (q_mgr >= cppi41_num_queue_mgr) |
| return -EINVAL; |
| |
| q_mgr_regs = cppi41_queue_mgr[q_mgr].q_mgr_rgn_base; |
| ptr = dma_alloc_coherent(NULL, rgn0_size * 4, |
| &linking_ram[q_mgr].phys_addr, |
| GFP_KERNEL | GFP_DMA); |
| if (ptr == NULL) { |
| printk(KERN_ERR "ERROR: %s: Unable to allocate " |
| "linking RAM.\n", __func__); |
| return -ENOMEM; |
| } |
| linking_ram[q_mgr].virt_addr = ptr; |
| linking_ram[q_mgr].size = rgn0_size * 4; |
| |
| __raw_writel(linking_ram[q_mgr].phys_addr, |
| q_mgr_regs + QMGR_LINKING_RAM_RGN0_BASE_REG); |
| DBG("Linking RAM region 0 base @ %p, value: %x\n", |
| q_mgr_regs + QMGR_LINKING_RAM_RGN0_BASE_REG, |
| __raw_readl(q_mgr_regs + QMGR_LINKING_RAM_RGN0_BASE_REG)); |
| |
| __raw_writel(rgn0_size, q_mgr_regs + QMGR_LINKING_RAM_RGN0_SIZE_REG); |
| DBG("Linking RAM region 0 size @ %p, value: %x\n", |
| q_mgr_regs + QMGR_LINKING_RAM_RGN0_SIZE_REG, |
| __raw_readl(q_mgr_regs + QMGR_LINKING_RAM_RGN0_SIZE_REG)); |
| |
| ptr = kzalloc(BITS_TO_LONGS(cppi41_queue_mgr[q_mgr].num_queue), |
| GFP_KERNEL); |
| if (ptr == NULL) { |
| printk(KERN_ERR "ERROR: %s: Unable to allocate queue bitmap.\n", |
| __func__); |
| dma_free_coherent(NULL, rgn0_size * 4, |
| linking_ram[q_mgr].virt_addr, |
| linking_ram[q_mgr].phys_addr); |
| return -ENOMEM; |
| } |
| allocated_queues[q_mgr] = ptr; |
| |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_queue_mgr_init); |
| |
| int cppi41_queue_mgr_uninit(u8 q_mgr) |
| { |
| void __iomem *q_mgr_regs; |
| |
| if (q_mgr >= cppi41_num_queue_mgr) |
| return -EINVAL; |
| |
| q_mgr_regs = cppi41_queue_mgr[q_mgr].q_mgr_rgn_base; |
| |
| /* free the Queue Mgr linking ram space */ |
| __raw_writel(0, q_mgr_regs + QMGR_LINKING_RAM_RGN0_BASE_REG); |
| __raw_writel(0, q_mgr_regs + QMGR_LINKING_RAM_RGN0_SIZE_REG); |
| dma_free_coherent(NULL, linking_ram[q_mgr].size, |
| linking_ram[q_mgr].virt_addr, |
| linking_ram[q_mgr].phys_addr); |
| |
| /* free the allocated queues */ |
| kfree(allocated_queues[q_mgr]); |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_queue_mgr_uninit); |
| |
| int cppi41_dma_sched_tbl_init(u8 dma_num, u8 q_mgr, |
| u32 *sched_tbl, u8 tbl_size) |
| { |
| struct cppi41_dma_block *dma_block; |
| int num_reg, k, i, val = 0; |
| |
| dma_block = (struct cppi41_dma_block *)&cppi41_dma_block[dma_num]; |
| |
| num_reg = (tbl_size + 3) / 4; |
| for (k = i = 0; i < num_reg; i++) { |
| #if 0 |
| for (val = j = 0; j < 4; j++, k++) { |
| val >>= 8; |
| if (k < tbl_size) |
| val |= sched_tbl[k] << 24; |
| } |
| #endif |
| val = sched_tbl[i]; |
| __raw_writel(val, dma_block->sched_table_base + |
| DMA_SCHED_TABLE_WORD_REG(i)); |
| DBG("DMA scheduler table @ %p, value written: %x\n", |
| dma_block->sched_table_base + DMA_SCHED_TABLE_WORD_REG(i), |
| val); |
| } |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_dma_sched_tbl_init); |
| |
| int cppi41_schedtbl_add_dma_ch(u8 dmanum, u8 qmgr, u8 dma_ch, u8 is_tx) |
| { |
| struct cppi41_dma_block *dma_block; |
| int num_ch, i, tbl_index = 0, j = 0, found = 0; |
| u32 val; |
| |
| dma_block = (struct cppi41_dma_block *)&cppi41_dma_block[dmanum]; |
| |
| val = 0; |
| for (num_ch = 0, i = 0; i < MAX_SCHED_TBL_ENTRY; i++) { |
| if (!found && dma_sched_tbl[i].dma_ch == dma_ch && |
| dma_sched_tbl[i].is_tx == is_tx && |
| dma_sched_tbl[i].enb == 0) { |
| dma_sched_tbl[i].enb = 1; |
| found = 1; |
| } |
| |
| if (dma_sched_tbl[i].enb) { |
| val |= ((dma_sched_tbl[i].dma_ch | |
| (dma_sched_tbl[i].is_tx ? 0 : (1<<7))) << j*8); |
| num_ch++; |
| j++; |
| } |
| if (num_ch % 4 == 0) { |
| __raw_writel(val, dma_block->sched_table_base + |
| DMA_SCHED_TABLE_WORD_REG(tbl_index)); |
| tbl_index++; |
| val = j = 0; |
| } |
| } |
| |
| if (num_ch % 4) { |
| __raw_writel(val, dma_block->sched_table_base + |
| DMA_SCHED_TABLE_WORD_REG(tbl_index)); |
| } |
| return num_ch; |
| } |
| EXPORT_SYMBOL(cppi41_schedtbl_add_dma_ch); |
| |
| int cppi41_schedtbl_remove_dma_ch(u8 dmanum, u8 qmgr, u8 dma_ch, u8 is_tx) |
| { |
| struct cppi41_dma_block *dma_block; |
| int num_ch, i, tbl_index = 0, j = 0, found = 0; |
| u32 val; |
| |
| dma_block = (struct cppi41_dma_block *)&cppi41_dma_block[dmanum]; |
| |
| val = 0; |
| for (num_ch = 0, i = 0; i < MAX_SCHED_TBL_ENTRY; i++) { |
| if (!found && dma_sched_tbl[i].dma_ch == dma_ch && |
| dma_sched_tbl[i].is_tx == is_tx && |
| dma_sched_tbl[i].enb == 1) { |
| dma_sched_tbl[i].enb = 0; |
| } |
| |
| if (dma_sched_tbl[i].enb) { |
| val |= ((dma_sched_tbl[i].dma_ch | |
| (dma_sched_tbl[i].is_tx ? 0 : (1<<7))) << j*8); |
| num_ch++; |
| j++; |
| } |
| if (num_ch % 4 == 0) { |
| __raw_writel(val, dma_block->sched_table_base + |
| DMA_SCHED_TABLE_WORD_REG(tbl_index)); |
| tbl_index++; |
| val = j = 0; |
| } |
| } |
| |
| if (num_ch % 4) { |
| __raw_writel(val, dma_block->sched_table_base + |
| DMA_SCHED_TABLE_WORD_REG(tbl_index)); |
| } |
| return num_ch; |
| } |
| EXPORT_SYMBOL(cppi41_schedtbl_remove_dma_ch); |
| |
| int cppi41_dma_block_init(u8 dma_num, u8 q_mgr, u8 num_order, |
| u32 *sched_tbl, u8 tbl_size) |
| { |
| const struct cppi41_dma_block *dma_block; |
| struct cppi41_teardown_desc *curr_td; |
| dma_addr_t td_addr; |
| unsigned num_desc, num_reg; |
| void *ptr; |
| int error, i; |
| u16 q_num; |
| u32 val; |
| |
| if (dma_num >= cppi41_num_dma_block || |
| q_mgr >= cppi41_num_queue_mgr || |
| !tbl_size || sched_tbl == NULL) |
| return -EINVAL; |
| |
| error = cppi41_queue_alloc(CPPI41_FREE_DESC_QUEUE | |
| CPPI41_UNASSIGNED_QUEUE, q_mgr, &q_num); |
| if (error) { |
| printk(KERN_ERR "ERROR: %s: Unable to allocate teardown " |
| "descriptor queue.\n", __func__); |
| return error; |
| } |
| DBG("Teardown descriptor queue %d in queue manager 0 " |
| "allocated\n", q_num); |
| |
| /* |
| * Tell the hardware about the Teardown descriptor |
| * queue manager and queue number. |
| */ |
| dma_block = &cppi41_dma_block[dma_num]; |
| __raw_writel((q_mgr << DMA_TD_DESC_QMGR_SHIFT) | |
| (q_num << DMA_TD_DESC_QNUM_SHIFT), |
| dma_block->global_ctrl_base + |
| DMA_TEARDOWN_FREE_DESC_CTRL_REG); |
| DBG("Teardown free descriptor control @ %p, value: %x\n", |
| dma_block->global_ctrl_base + DMA_TEARDOWN_FREE_DESC_CTRL_REG, |
| __raw_readl(dma_block->global_ctrl_base + |
| DMA_TEARDOWN_FREE_DESC_CTRL_REG)); |
| |
| num_desc = 1 << num_order; |
| dma_teardown[dma_num].rgn_size = num_desc * |
| sizeof(struct cppi41_teardown_desc); |
| |
| /* Pre-allocate teardown descriptors. */ |
| ptr = dma_alloc_coherent(NULL, dma_teardown[dma_num].rgn_size, |
| &dma_teardown[dma_num].phys_addr, |
| GFP_KERNEL | GFP_DMA); |
| if (ptr == NULL) { |
| printk(KERN_ERR "ERROR: %s: Unable to allocate teardown " |
| "descriptors.\n", __func__); |
| error = -ENOMEM; |
| goto free_queue; |
| } |
| dma_teardown[dma_num].virt_addr = ptr; |
| |
| error = cppi41_mem_rgn_alloc(q_mgr, dma_teardown[dma_num].phys_addr, 5, |
| num_order, &dma_teardown[dma_num].mem_rgn); |
| if (error) { |
| printk(KERN_ERR "ERROR: %s: Unable to allocate queue manager " |
| "memory region for teardown descriptors.\n", __func__); |
| goto free_mem; |
| } |
| |
| error = cppi41_queue_init(&dma_teardown[dma_num].queue_obj, 0, q_num); |
| if (error) { |
| printk(KERN_ERR "ERROR: %s: Unable to initialize teardown " |
| "free descriptor queue.\n", __func__); |
| goto free_rgn; |
| } |
| |
| dma_teardown[dma_num].q_num = q_num; |
| dma_teardown[dma_num].q_mgr = q_mgr; |
| /* |
| * Push all teardown descriptors to the free teardown queue |
| * for the CPPI 4.1 system. |
| */ |
| curr_td = dma_teardown[dma_num].virt_addr; |
| td_addr = dma_teardown[dma_num].phys_addr; |
| |
| for (i = 0; i < num_desc; i++) { |
| cppi41_queue_push(&dma_teardown[dma_num].queue_obj, td_addr, |
| sizeof(*curr_td), 0); |
| td_addr += sizeof(*curr_td); |
| } |
| |
| /* Initialize the DMA scheduler. */ |
| num_reg = (tbl_size + 3) / 4; |
| for (i = 0; i < num_reg; i++) { |
| val = sched_tbl[i]; |
| __raw_writel(val, dma_block->sched_table_base + |
| DMA_SCHED_TABLE_WORD_REG(i)); |
| DBG("DMA scheduler table @ %p, value written: %x\n", |
| dma_block->sched_table_base + DMA_SCHED_TABLE_WORD_REG(i), |
| val); |
| } |
| |
| __raw_writel((tbl_size - 1) << DMA_SCHED_LAST_ENTRY_SHIFT | |
| DMA_SCHED_ENABLE_MASK, |
| dma_block->sched_ctrl_base + DMA_SCHED_CTRL_REG); |
| DBG("DMA scheduler control @ %p, value: %x\n", |
| dma_block->sched_ctrl_base + DMA_SCHED_CTRL_REG, |
| __raw_readl(dma_block->sched_ctrl_base + DMA_SCHED_CTRL_REG)); |
| |
| return 0; |
| |
| free_rgn: |
| cppi41_mem_rgn_free(q_mgr, dma_teardown[dma_num].mem_rgn); |
| free_mem: |
| dma_free_coherent(NULL, dma_teardown[dma_num].rgn_size, |
| dma_teardown[dma_num].virt_addr, |
| dma_teardown[dma_num].phys_addr); |
| free_queue: |
| cppi41_queue_free(q_mgr, q_num); |
| return error; |
| } |
| EXPORT_SYMBOL(cppi41_dma_block_init); |
| |
| int cppi41_dma_block_uninit(u8 dma_num, u8 q_mgr, u8 num_order, |
| u32 *sched_tbl, u8 tbl_size) |
| { |
| const struct cppi41_dma_block *dma_block; |
| unsigned num_reg; |
| int i; |
| |
| /* popout all teardown descriptors */ |
| cppi41_free_teardown_queue(dma_num); |
| |
| /* free queue mgr region */ |
| cppi41_mem_rgn_free(q_mgr, dma_teardown[dma_num].mem_rgn); |
| /* free the allocated teardown descriptors */ |
| dma_free_coherent(NULL, dma_teardown[dma_num].rgn_size, |
| dma_teardown[dma_num].virt_addr, |
| dma_teardown[dma_num].phys_addr); |
| |
| /* free the teardown queue*/ |
| cppi41_queue_free(dma_teardown[dma_num].q_mgr, |
| dma_teardown[dma_num].q_num); |
| |
| dma_block = (struct cppi41_dma_block *)&cppi41_dma_block[dma_num]; |
| /* disable the dma schedular */ |
| num_reg = (tbl_size + 3) / 4; |
| for (i = 0; i < num_reg; i++) { |
| __raw_writel(0, dma_block->sched_table_base + |
| DMA_SCHED_TABLE_WORD_REG(i)); |
| DBG("DMA scheduler table @ %p, value written: %x\n", |
| dma_block->sched_table_base + DMA_SCHED_TABLE_WORD_REG(i), |
| 0); |
| } |
| |
| __raw_writel(0, dma_block->sched_ctrl_base + DMA_SCHED_CTRL_REG); |
| |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_dma_block_uninit); |
| /* |
| * cppi41_mem_rgn_alloc - allocate a memory region within the queue manager |
| */ |
| int cppi41_mem_rgn_alloc(u8 q_mgr, dma_addr_t rgn_addr, u8 size_order, |
| u8 num_order, u8 *mem_rgn) |
| { |
| void __iomem *desc_mem_regs; |
| u32 num_desc = 1 << num_order, index, ctrl; |
| int rgn; |
| |
| DBG("%s called with rgn_addr = %08x, size_order = %d, num_order = %d\n", |
| __func__, rgn_addr, size_order, num_order); |
| |
| if (q_mgr >= cppi41_num_queue_mgr || |
| size_order < 5 || size_order > 13 || |
| num_order < 5 || num_order > 12 || |
| (rgn_addr & ((1 << size_order) - 1))) |
| return -EINVAL; |
| |
| rgn = next_mem_rgn[q_mgr]; |
| index = next_desc_index[q_mgr]; |
| if (rgn >= CPPI41_MAX_MEM_RGN || index + num_desc > 0x4000) |
| return -ENOSPC; |
| |
| next_mem_rgn[q_mgr] = rgn + 1; |
| next_desc_index[q_mgr] = index + num_desc; |
| |
| desc_mem_regs = cppi41_queue_mgr[q_mgr].desc_mem_rgn_base; |
| |
| /* Write the base register */ |
| __raw_writel(rgn_addr, desc_mem_regs + QMGR_MEM_RGN_BASE_REG(rgn)); |
| DBG("Descriptor region base @ %p, value: %x\n", |
| desc_mem_regs + QMGR_MEM_RGN_BASE_REG(rgn), |
| __raw_readl(desc_mem_regs + QMGR_MEM_RGN_BASE_REG(rgn))); |
| |
| /* Write the control register */ |
| ctrl = ((index << QMGR_MEM_RGN_INDEX_SHIFT) & |
| QMGR_MEM_RGN_INDEX_MASK) | |
| (((size_order - 5) << QMGR_MEM_RGN_DESC_SIZE_SHIFT) & |
| QMGR_MEM_RGN_DESC_SIZE_MASK) | |
| (((num_order - 5) << QMGR_MEM_RGN_SIZE_SHIFT) & |
| QMGR_MEM_RGN_SIZE_MASK); |
| __raw_writel(ctrl, desc_mem_regs + QMGR_MEM_RGN_CTRL_REG(rgn)); |
| DBG("Descriptor region control @ %p, value: %x\n", |
| desc_mem_regs + QMGR_MEM_RGN_CTRL_REG(rgn), |
| __raw_readl(desc_mem_regs + QMGR_MEM_RGN_CTRL_REG(rgn))); |
| |
| *mem_rgn = rgn; |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_mem_rgn_alloc); |
| |
| /* |
| * cppi41_mem_rgn_free - free the memory region within the queue manager |
| */ |
| int cppi41_mem_rgn_free(u8 q_mgr, u8 mem_rgn) |
| { |
| void __iomem *desc_mem_regs; |
| |
| DBG("%s called.\n", __func__); |
| |
| if (q_mgr >= cppi41_num_queue_mgr || mem_rgn >= next_mem_rgn[q_mgr]) |
| return -EINVAL; |
| |
| desc_mem_regs = cppi41_queue_mgr[q_mgr].desc_mem_rgn_base; |
| |
| if (__raw_readl(desc_mem_regs + QMGR_MEM_RGN_BASE_REG(mem_rgn)) == 0) |
| return -ENOENT; |
| |
| __raw_writel(0, desc_mem_regs + QMGR_MEM_RGN_BASE_REG(mem_rgn)); |
| __raw_writel(0, desc_mem_regs + QMGR_MEM_RGN_CTRL_REG(mem_rgn)); |
| |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_mem_rgn_free); |
| |
| /* |
| * cppi41_tx_ch_init - initialize a CPPI 4.1 Tx channel object |
| * |
| * Verify the channel info (range checking, etc.) and store the channel |
| * information within the object structure. |
| */ |
| int cppi41_tx_ch_init(struct cppi41_dma_ch_obj *tx_ch_obj, |
| u8 dma_num, u8 ch_num) |
| { |
| if (dma_num >= cppi41_num_dma_block || |
| ch_num >= cppi41_dma_block[dma_num].num_tx_ch) |
| return -EINVAL; |
| |
| /* Populate the channel object structure */ |
| tx_ch_obj->base_addr = cppi41_dma_block[dma_num].ch_ctrl_stat_base + |
| DMA_CH_TX_GLOBAL_CFG_REG(ch_num); |
| tx_ch_obj->global_cfg = __raw_readl(tx_ch_obj->base_addr); |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_tx_ch_init); |
| |
| /* |
| * cppi41_rx_ch_init - initialize a CPPI 4.1 Rx channel object |
| * |
| * Verify the channel info (range checking, etc.) and store the channel |
| * information within the object structure. |
| */ |
| int cppi41_rx_ch_init(struct cppi41_dma_ch_obj *rx_ch_obj, |
| u8 dma_num, u8 ch_num) |
| { |
| if (dma_num >= cppi41_num_dma_block || |
| ch_num >= cppi41_dma_block[dma_num].num_rx_ch) |
| return -EINVAL; |
| |
| /* Populate the channel object structure */ |
| rx_ch_obj->base_addr = cppi41_dma_block[dma_num].ch_ctrl_stat_base + |
| DMA_CH_RX_GLOBAL_CFG_REG(ch_num); |
| rx_ch_obj->global_cfg = __raw_readl(rx_ch_obj->base_addr); |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_rx_ch_init); |
| |
| /* |
| * We have to cache the last written Rx/Tx channel global configration register |
| * value due to its bits other than enable/teardown being write-only. Yet there |
| * is a caveat related to caching the enable bit: this bit may be automatically |
| * cleared as a result of teardown, so we can't trust its cached value! |
| * When modifying the write only register fields, we're making use of the fact |
| * that they read back as zeros, and not clearing them explicitly... |
| */ |
| |
| /* |
| * cppi41_dma_ch_default_queue - set CPPI 4.1 channel default completion queue |
| */ |
| void cppi41_dma_ch_default_queue(struct cppi41_dma_ch_obj *dma_ch_obj, |
| u8 q_mgr, u16 q_num) |
| { |
| u32 val = dma_ch_obj->global_cfg; |
| |
| /* Clear the fields to be modified. */ |
| val &= ~(DMA_CH_TX_DEFAULT_QMGR_MASK | DMA_CH_TX_DEFAULT_QNUM_MASK | |
| DMA_CH_TX_ENABLE_MASK); |
| |
| /* Set the default completion queue. */ |
| val |= ((q_mgr << DMA_CH_TX_DEFAULT_QMGR_SHIFT) & |
| DMA_CH_TX_DEFAULT_QMGR_MASK) | |
| ((q_num << DMA_CH_TX_DEFAULT_QNUM_SHIFT) & |
| DMA_CH_TX_DEFAULT_QNUM_MASK); |
| |
| /* Get the current state of the enable bit. */ |
| dma_ch_obj->global_cfg = val |= __raw_readl(dma_ch_obj->base_addr); |
| __raw_writel(val, dma_ch_obj->base_addr); |
| DBG("Channel global configuration @ %p, value written: %x, " |
| "value read: %x\n", dma_ch_obj->base_addr, val, |
| __raw_readl(dma_ch_obj->base_addr)); |
| |
| } |
| EXPORT_SYMBOL(cppi41_dma_ch_default_queue); |
| |
| /* |
| * cppi41_rx_ch_configure - configure CPPI 4.1 Rx channel |
| */ |
| void cppi41_rx_ch_configure(struct cppi41_dma_ch_obj *rx_ch_obj, |
| struct cppi41_rx_ch_cfg *cfg) |
| { |
| void __iomem *base = rx_ch_obj->base_addr; |
| u32 val = __raw_readl(rx_ch_obj->base_addr); |
| |
| val |= ((cfg->sop_offset << DMA_CH_RX_SOP_OFFSET_SHIFT) & |
| DMA_CH_RX_SOP_OFFSET_MASK) | |
| ((cfg->default_desc_type << DMA_CH_RX_DEFAULT_DESC_TYPE_SHIFT) & |
| DMA_CH_RX_DEFAULT_DESC_TYPE_MASK) | |
| ((cfg->retry_starved << DMA_CH_RX_ERROR_HANDLING_SHIFT) & |
| DMA_CH_RX_ERROR_HANDLING_MASK) | |
| ((cfg->rx_queue.q_mgr << DMA_CH_RX_DEFAULT_RQ_QMGR_SHIFT) & |
| DMA_CH_RX_DEFAULT_RQ_QMGR_MASK) | |
| ((cfg->rx_queue.q_num << DMA_CH_RX_DEFAULT_RQ_QNUM_SHIFT) & |
| DMA_CH_RX_DEFAULT_RQ_QNUM_MASK); |
| |
| rx_ch_obj->global_cfg = val; |
| __raw_writel(val, base); |
| DBG("Rx channel global configuration @ %p, value written: %x, " |
| "value read: %x\n", base, val, __raw_readl(base)); |
| |
| base -= DMA_CH_RX_GLOBAL_CFG_REG(0); |
| |
| /* |
| * Set up the packet configuration register |
| * based on the descriptor type... |
| */ |
| switch (cfg->default_desc_type) { |
| case DMA_CH_RX_DEFAULT_DESC_EMBED: |
| val = ((cfg->cfg.embed_pkt.fd_queue.q_mgr << |
| DMA_CH_RX_EMBED_FDQ_QMGR_SHIFT) & |
| DMA_CH_RX_EMBED_FDQ_QMGR_MASK) | |
| ((cfg->cfg.embed_pkt.fd_queue.q_num << |
| DMA_CH_RX_EMBED_FDQ_QNUM_SHIFT) & |
| DMA_CH_RX_EMBED_FDQ_QNUM_MASK) | |
| ((cfg->cfg.embed_pkt.num_buf_slot << |
| DMA_CH_RX_EMBED_NUM_SLOT_SHIFT) & |
| DMA_CH_RX_EMBED_NUM_SLOT_MASK) | |
| ((cfg->cfg.embed_pkt.sop_slot_num << |
| DMA_CH_RX_EMBED_SOP_SLOT_SHIFT) & |
| DMA_CH_RX_EMBED_SOP_SLOT_MASK); |
| |
| __raw_writel(val, base + DMA_CH_RX_EMBED_PKT_CFG_REG_B(0)); |
| DBG("Rx channel embedded packet configuration B @ %p, " |
| "value written: %x\n", |
| base + DMA_CH_RX_EMBED_PKT_CFG_REG_B(0), val); |
| |
| val = ((cfg->cfg.embed_pkt.free_buf_pool[0].b_pool << |
| DMA_CH_RX_EMBED_FBP_PNUM_SHIFT(0)) & |
| DMA_CH_RX_EMBED_FBP_PNUM_MASK(0)) | |
| ((cfg->cfg.embed_pkt.free_buf_pool[0].b_mgr << |
| DMA_CH_RX_EMBED_FBP_BMGR_SHIFT(0)) & |
| DMA_CH_RX_EMBED_FBP_BMGR_MASK(0)) | |
| ((cfg->cfg.embed_pkt.free_buf_pool[1].b_pool << |
| DMA_CH_RX_EMBED_FBP_PNUM_SHIFT(1)) & |
| DMA_CH_RX_EMBED_FBP_PNUM_MASK(1)) | |
| ((cfg->cfg.embed_pkt.free_buf_pool[1].b_mgr << |
| DMA_CH_RX_EMBED_FBP_BMGR_SHIFT(1)) & |
| DMA_CH_RX_EMBED_FBP_BMGR_MASK(1)) | |
| ((cfg->cfg.embed_pkt.free_buf_pool[2].b_pool << |
| DMA_CH_RX_EMBED_FBP_PNUM_SHIFT(2)) & |
| DMA_CH_RX_EMBED_FBP_PNUM_MASK(2)) | |
| ((cfg->cfg.embed_pkt.free_buf_pool[2].b_mgr << |
| DMA_CH_RX_EMBED_FBP_BMGR_SHIFT(2)) & |
| DMA_CH_RX_EMBED_FBP_BMGR_MASK(2)) | |
| ((cfg->cfg.embed_pkt.free_buf_pool[3].b_pool << |
| DMA_CH_RX_EMBED_FBP_PNUM_SHIFT(3)) & |
| DMA_CH_RX_EMBED_FBP_PNUM_MASK(3)) | |
| ((cfg->cfg.embed_pkt.free_buf_pool[3].b_mgr << |
| DMA_CH_RX_EMBED_FBP_BMGR_SHIFT(3)) & |
| DMA_CH_RX_EMBED_FBP_BMGR_MASK(3)); |
| |
| __raw_writel(val, base + DMA_CH_RX_EMBED_PKT_CFG_REG_A(0)); |
| DBG("Rx channel embedded packet configuration A @ %p, " |
| "value written: %x\n", |
| base + DMA_CH_RX_EMBED_PKT_CFG_REG_A(0), val); |
| break; |
| case DMA_CH_RX_DEFAULT_DESC_HOST: |
| val = ((cfg->cfg.host_pkt.fdb_queue[0].q_num << |
| DMA_CH_RX_HOST_FDQ_QNUM_SHIFT(0)) & |
| DMA_CH_RX_HOST_FDQ_QNUM_MASK(0)) | |
| ((cfg->cfg.host_pkt.fdb_queue[0].q_mgr << |
| DMA_CH_RX_HOST_FDQ_QMGR_SHIFT(0)) & |
| DMA_CH_RX_HOST_FDQ_QMGR_MASK(0)) | |
| ((cfg->cfg.host_pkt.fdb_queue[1].q_num << |
| DMA_CH_RX_HOST_FDQ_QNUM_SHIFT(1)) & |
| DMA_CH_RX_HOST_FDQ_QNUM_MASK(1)) | |
| ((cfg->cfg.host_pkt.fdb_queue[1].q_mgr << |
| DMA_CH_RX_HOST_FDQ_QMGR_SHIFT(1)) & |
| DMA_CH_RX_HOST_FDQ_QMGR_MASK(1)); |
| |
| __raw_writel(val, base + DMA_CH_RX_HOST_PKT_CFG_REG_A(0)); |
| DBG("Rx channel host packet configuration A @ %p, " |
| "value written: %x\n", |
| base + DMA_CH_RX_HOST_PKT_CFG_REG_A(0), val); |
| |
| val = ((cfg->cfg.host_pkt.fdb_queue[2].q_num << |
| DMA_CH_RX_HOST_FDQ_QNUM_SHIFT(2)) & |
| DMA_CH_RX_HOST_FDQ_QNUM_MASK(2)) | |
| ((cfg->cfg.host_pkt.fdb_queue[2].q_mgr << |
| DMA_CH_RX_HOST_FDQ_QMGR_SHIFT(2)) & |
| DMA_CH_RX_HOST_FDQ_QMGR_MASK(2)) | |
| ((cfg->cfg.host_pkt.fdb_queue[3].q_num << |
| DMA_CH_RX_HOST_FDQ_QNUM_SHIFT(3)) & |
| DMA_CH_RX_HOST_FDQ_QNUM_MASK(3)) | |
| ((cfg->cfg.host_pkt.fdb_queue[3].q_mgr << |
| DMA_CH_RX_HOST_FDQ_QMGR_SHIFT(3)) & |
| DMA_CH_RX_HOST_FDQ_QMGR_MASK(3)); |
| |
| __raw_writel(val, base + DMA_CH_RX_HOST_PKT_CFG_REG_B(0)); |
| DBG("Rx channel host packet configuration B @ %p, " |
| "value written: %x\n", |
| base + DMA_CH_RX_HOST_PKT_CFG_REG_B(0), val); |
| break; |
| case DMA_CH_RX_DEFAULT_DESC_MONO: |
| val = ((cfg->cfg.mono_pkt.fd_queue.q_num << |
| DMA_CH_RX_MONO_FDQ_QNUM_SHIFT) & |
| DMA_CH_RX_MONO_FDQ_QNUM_MASK) | |
| ((cfg->cfg.mono_pkt.fd_queue.q_mgr << |
| DMA_CH_RX_MONO_FDQ_QMGR_SHIFT) & |
| DMA_CH_RX_MONO_FDQ_QMGR_MASK) | |
| ((cfg->cfg.mono_pkt.sop_offset << |
| DMA_CH_RX_MONO_SOP_OFFSET_SHIFT) & |
| DMA_CH_RX_MONO_SOP_OFFSET_MASK); |
| |
| __raw_writel(val, base + DMA_CH_RX_MONO_PKT_CFG_REG(0)); |
| DBG("Rx channel monolithic packet configuration @ %p, " |
| "value written: %x\n", |
| base + DMA_CH_RX_MONO_PKT_CFG_REG(0), val); |
| break; |
| } |
| } |
| EXPORT_SYMBOL(cppi41_rx_ch_configure); |
| |
| /* |
| * cppi41_dma_ch_teardown - teardown a given Tx/Rx channel |
| */ |
| void cppi41_dma_ch_teardown(struct cppi41_dma_ch_obj *dma_ch_obj) |
| { |
| u32 val = __raw_readl(dma_ch_obj->base_addr); |
| |
| /* Initiate channel teardown. */ |
| val |= dma_ch_obj->global_cfg & ~DMA_CH_TX_ENABLE_MASK; |
| dma_ch_obj->global_cfg = val |= DMA_CH_TX_TEARDOWN_MASK; |
| __raw_writel(val, dma_ch_obj->base_addr); |
| DBG("Tear down channel @ %p, value written: %x, value read: %x\n", |
| dma_ch_obj->base_addr, val, __raw_readl(dma_ch_obj->base_addr)); |
| } |
| EXPORT_SYMBOL(cppi41_dma_ch_teardown); |
| |
| /* |
| * cppi41_dma_ch_enable - enable Tx/Rx DMA channel in hardware |
| * |
| * Makes the channel ready for data transmission/reception. |
| */ |
| void cppi41_dma_ch_enable(struct cppi41_dma_ch_obj *dma_ch_obj) |
| { |
| u32 val = dma_ch_obj->global_cfg | DMA_CH_TX_ENABLE_MASK; |
| |
| /* Teardown bit remains set after completion, so clear it now... */ |
| dma_ch_obj->global_cfg = val &= ~DMA_CH_TX_TEARDOWN_MASK; |
| __raw_writel(val, dma_ch_obj->base_addr); |
| DBG("Enable channel @ %p, value written: %x, value read: %x\n", |
| dma_ch_obj->base_addr, val, __raw_readl(dma_ch_obj->base_addr)); |
| } |
| EXPORT_SYMBOL(cppi41_dma_ch_enable); |
| |
| /* |
| * cppi41_dma_ch_disable - disable Tx/Rx DMA channel in hardware |
| */ |
| void cppi41_dma_ch_disable(struct cppi41_dma_ch_obj *dma_ch_obj) |
| { |
| dma_ch_obj->global_cfg &= ~DMA_CH_TX_ENABLE_MASK; |
| __raw_writel(dma_ch_obj->global_cfg, dma_ch_obj->base_addr); |
| DBG("Disable channel @ %p, value written: %x, value read: %x\n", |
| dma_ch_obj->base_addr, dma_ch_obj->global_cfg, |
| __raw_readl(dma_ch_obj->base_addr)); |
| } |
| EXPORT_SYMBOL(cppi41_dma_ch_disable); |
| |
| void cppi41_free_teardown_queue(int dma_num) |
| { |
| unsigned long td_addr; |
| |
| while (1) { |
| td_addr = cppi41_queue_pop(&dma_teardown[dma_num].queue_obj); |
| |
| if (td_addr == 0) |
| break; |
| } |
| } |
| EXPORT_SYMBOL(cppi41_free_teardown_queue); |
| |
| /** |
| * alloc_queue - allocate a queue in the given range |
| * @allocated: pointer to the bitmap of the allocated queues |
| * @excluded: pointer to the bitmap of the queues excluded from allocation |
| * (optional) |
| * @start: starting queue number |
| * @count: number of queues available |
| * |
| * Returns queue number on success, -ENOSPC otherwise. |
| */ |
| static int alloc_queue(u32 *allocated, const u32 *excluded, unsigned start, |
| unsigned count) |
| { |
| u32 bit, mask = 0; |
| int index = -1; |
| |
| /* |
| * We're starting the loop as if we've just wrapped around 32 bits |
| * in order to save on preloading the bitmasks. |
| */ |
| for (bit = 0; count--; start++, bit <<= 1) { |
| /* Have we just wrapped around 32 bits? */ |
| if (!bit) { |
| /* Start over with the next bitmask word */ |
| bit = 1; |
| index++; |
| /* Have we just entered the loop? */ |
| if (!index) { |
| /* Calculate the starting values */ |
| bit <<= start & 0x1f; |
| index = start >> 5; |
| } |
| /* |
| * Load the next word of the allocated bitmask OR'ing |
| * it with the excluded bitmask if it's been passed. |
| */ |
| mask = allocated[index]; |
| if (excluded != NULL) |
| mask |= excluded[index]; |
| } |
| /* |
| * If the bit in the combined bitmask is zero, |
| * we've just found a free queue. |
| */ |
| if (!(mask & bit)) { |
| allocated[index] |= bit; |
| return start; |
| } |
| } |
| return -ENOSPC; |
| } |
| |
| /* |
| * cppi41_queue_alloc - allocate a queue of a given type in the queue manager |
| */ |
| int cppi41_queue_alloc(u8 type, u8 q_mgr, u16 *q_num) |
| { |
| int res = -ENOSPC; |
| |
| if (q_mgr >= cppi41_num_queue_mgr) |
| return -EINVAL; |
| |
| /* Mask out the unsupported queue types */ |
| type &= cppi41_queue_mgr[q_mgr].queue_types; |
| /* First see if a free descriptor queue was requested... */ |
| if (type & CPPI41_FREE_DESC_QUEUE) |
| res = alloc_queue(allocated_queues[q_mgr], NULL, |
| cppi41_queue_mgr[q_mgr].base_fdq_num, 16); |
| |
| /* Then see if a free descriptor/buffer queue was requested... */ |
| if (res < 0 && (type & CPPI41_FREE_DESC_BUF_QUEUE)) |
| res = alloc_queue(allocated_queues[q_mgr], NULL, |
| cppi41_queue_mgr[q_mgr].base_fdbq_num, 16); |
| |
| /* Last see if an unassigned queue was requested... */ |
| if (res < 0 && (type & CPPI41_UNASSIGNED_QUEUE)) |
| res = alloc_queue(allocated_queues[q_mgr], |
| cppi41_queue_mgr[q_mgr].assigned, 0, |
| cppi41_queue_mgr[q_mgr].num_queue); |
| |
| /* See if any queue was allocated... */ |
| if (res < 0) |
| return res; |
| |
| /* Return the queue allocated */ |
| *q_num = res; |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_queue_alloc); |
| |
| /* |
| * cppi41_queue_free - free the given queue in the queue manager |
| */ |
| int cppi41_queue_free(u8 q_mgr, u16 q_num) |
| { |
| int index = q_num >> 5, bit = 1 << (q_num & 0x1f); |
| |
| if (allocated_queues[q_mgr] != NULL) { |
| if (q_mgr >= cppi41_num_queue_mgr || |
| q_num >= cppi41_queue_mgr[q_mgr].num_queue || |
| !(allocated_queues[q_mgr][index] & bit)) |
| return -EINVAL; |
| allocated_queues[q_mgr][index] &= ~bit; |
| } |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_queue_free); |
| |
| /* |
| * cppi41_queue_init - initialize a CPPI 4.1 queue object |
| */ |
| int cppi41_queue_init(struct cppi41_queue_obj *queue_obj, u8 q_mgr, u16 q_num) |
| { |
| if (q_mgr >= cppi41_num_queue_mgr || |
| q_num >= cppi41_queue_mgr[q_mgr].num_queue) |
| return -EINVAL; |
| |
| queue_obj->base_addr = cppi41_queue_mgr[q_mgr].q_mgmt_rgn_base + |
| QMGR_QUEUE_STATUS_REG_A(q_num); |
| |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_queue_init); |
| |
| /* |
| * cppi41_queue_push - push a descriptor into the given queue |
| */ |
| void cppi41_queue_push(const struct cppi41_queue_obj *queue_obj, u32 desc_addr, |
| u32 desc_size, u32 pkt_size) |
| { |
| u32 val; |
| |
| /* |
| * Write to the tail of the queue. |
| * TODO: Can't think of a reason why a queue to head may be required. |
| * If it is, the API may have to be extended. |
| */ |
| #if 0 |
| /* |
| * Also, can't understand why packet size is required to queue up a |
| * descriptor. The spec says packet size *must* be written prior to |
| * the packet write operation. |
| */ |
| if (pkt_size) |
| val = (pkt_size << QMGR_QUEUE_PKT_SIZE_SHIFT) & |
| QMGR_QUEUE_PKT_SIZE_MASK; |
| __raw_writel(val, queue_obj->base_addr + QMGR_QUEUE_REG_C(0)); |
| #endif |
| |
| val = (((desc_size - 24) >> (2 - QMGR_QUEUE_DESC_SIZE_SHIFT)) & |
| QMGR_QUEUE_DESC_SIZE_MASK) | |
| (desc_addr & QMGR_QUEUE_DESC_PTR_MASK); |
| |
| DBG("Pushing value %x to queue @ %p\n", val, queue_obj->base_addr); |
| |
| __raw_writel(val, queue_obj->base_addr + QMGR_QUEUE_REG_D(0)); |
| } |
| EXPORT_SYMBOL(cppi41_queue_push); |
| |
| /* |
| * cppi41_queue_pop - pop a descriptor from a given queue |
| */ |
| unsigned long cppi41_queue_pop(const struct cppi41_queue_obj *queue_obj) |
| { |
| u32 val = __raw_readl(queue_obj->base_addr + QMGR_QUEUE_REG_D(0)); |
| |
| DBG("Popping value %x from queue @ %p\n", val, queue_obj->base_addr); |
| |
| return val & QMGR_QUEUE_DESC_PTR_MASK; |
| } |
| EXPORT_SYMBOL(cppi41_queue_pop); |
| |
| /* |
| * cppi41_get_teardown_info - extract information from a teardown descriptor |
| */ |
| int cppi41_get_teardown_info(unsigned long addr, u32 *info) |
| { |
| struct cppi41_teardown_desc *desc; |
| int dma_num; |
| |
| for (dma_num = 0; dma_num < cppi41_num_dma_block; dma_num++) |
| if (addr >= dma_teardown[dma_num].phys_addr && |
| addr < dma_teardown[dma_num].phys_addr + |
| dma_teardown[dma_num].rgn_size) |
| break; |
| |
| if (dma_num == cppi41_num_dma_block) |
| return -EINVAL; |
| |
| desc = addr - dma_teardown[dma_num].phys_addr + |
| dma_teardown[dma_num].virt_addr; |
| |
| if ((desc->teardown_info & CPPI41_DESC_TYPE_MASK) != |
| (CPPI41_DESC_TYPE_TEARDOWN << CPPI41_DESC_TYPE_SHIFT)) |
| return -EINVAL; |
| |
| *info = desc->teardown_info; |
| #if 1 |
| /* Hardware is not giving the current DMA number as of now. :-/ */ |
| *info |= (dma_num << CPPI41_TEARDOWN_DMA_NUM_SHIFT) & |
| CPPI41_TEARDOWN_DMA_NUM_MASK; |
| #else |
| dma_num = (desc->teardown_info & CPPI41_TEARDOWN_DMA_NUM_MASK) >> |
| CPPI41_TEARDOWN_DMA_NUM_SHIFT; |
| #endif |
| |
| cppi41_queue_push(&dma_teardown[dma_num].queue_obj, addr, |
| sizeof(struct cppi41_teardown_desc), 0); |
| |
| return 0; |
| } |
| EXPORT_SYMBOL(cppi41_get_teardown_info); |
| |
| MODULE_DESCRIPTION("TI CPPI 4.1 support"); |
| MODULE_AUTHOR("MontaVista Software"); |
| MODULE_LICENSE("GPL"); |