| /* |
| * tiler-main.c |
| * |
| * TILER driver main support functions for TI TILER hardware block. |
| * |
| * Authors: Lajos Molnar <molnar@ti.com> |
| * David Sin <davidsin@ti.com> |
| * |
| * Copyright (C) 2009-2010 Texas Instruments, Inc. |
| * |
| * This package is free software; you can redistribute it and/or modify |
| * it under the terms of the GNU General Public License version 2 as |
| * published by the Free Software Foundation. |
| * |
| * THIS PACKAGE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR |
| * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED |
| * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. |
| */ |
| |
| #include <linux/init.h> |
| #include <linux/module.h> |
| #include <linux/cdev.h> /* struct cdev */ |
| #include <linux/kdev_t.h> /* MKDEV() */ |
| #include <linux/sched.h> /* current */ |
| #include <linux/fs.h> /* register_chrdev_region() */ |
| #include <linux/device.h> /* struct class */ |
| #include <linux/platform_device.h> /* platform_device() */ |
| #include <linux/err.h> /* IS_ERR() */ |
| #include <linux/errno.h> |
| #include <linux/mutex.h> |
| #include <linux/dma-mapping.h> /* dma_alloc_coherent */ |
| #include <linux/pagemap.h> /* page_cache_release() */ |
| #include <linux/slab.h> |
| |
| #include <mach/dmm.h> |
| #include "tmm.h" |
| #include "_tiler.h" |
| #include "tcm/tcm-sita.h" /* TCM algorithm */ |
| |
| static bool ssptr_id = CONFIG_TILER_SSPTR_ID; |
| static uint default_align = CONFIG_TILER_ALIGNMENT; |
| static uint granularity = CONFIG_TILER_GRANULARITY; |
| |
| /* |
| * We can only change ssptr_id if there are no blocks allocated, so that |
| * pseudo-random ids and ssptrs do not potentially clash. For now make it |
| * read-only. |
| */ |
| module_param(ssptr_id, bool, 0444); |
| MODULE_PARM_DESC(ssptr_id, "Use ssptr as block ID"); |
| module_param_named(align, default_align, uint, 0644); |
| MODULE_PARM_DESC(align, "Default block ssptr alignment"); |
| module_param_named(grain, granularity, uint, 0644); |
| MODULE_PARM_DESC(grain, "Granularity (bytes)"); |
| |
| struct tiler_dev { |
| struct cdev cdev; |
| }; |
| |
| struct platform_driver tiler_driver_ldm = { |
| .driver = { |
| .owner = THIS_MODULE, |
| .name = "tiler", |
| }, |
| .probe = NULL, |
| .shutdown = NULL, |
| .remove = NULL, |
| }; |
| |
| static struct tiler_ops tiler; /* shared methods and variables */ |
| |
| static struct list_head blocks; /* all tiler blocks */ |
| static struct list_head orphan_areas; /* orphaned 2D areas */ |
| static struct list_head orphan_onedim; /* orphaned 1D areas */ |
| |
| static s32 tiler_major; |
| static s32 tiler_minor; |
| static struct tiler_dev *tiler_device; |
| static struct class *tilerdev_class; |
| static struct mutex mtx; |
| static struct tcm *tcm[TILER_FORMATS]; |
| static struct tmm *tmm[TILER_FORMATS]; |
| static u32 *dmac_va; |
| static dma_addr_t dmac_pa; |
| |
| /* |
| * TMM connectors |
| * ========================================================================== |
| */ |
| /* wrapper around tmm_map */ |
| static s32 refill_pat(struct tmm *tmm, struct tcm_area *area, u32 *ptr) |
| { |
| s32 res = 0; |
| struct pat_area p_area = {0}; |
| struct tcm_area slice, area_s; |
| |
| tcm_for_each_slice(slice, *area, area_s) { |
| p_area.x0 = slice.p0.x; |
| p_area.y0 = slice.p0.y; |
| p_area.x1 = slice.p1.x; |
| p_area.y1 = slice.p1.y; |
| |
| memcpy(dmac_va, ptr, sizeof(*ptr) * tcm_sizeof(slice)); |
| ptr += tcm_sizeof(slice); |
| |
| if (tmm_map(tmm, p_area, dmac_pa)) { |
| res = -EFAULT; |
| break; |
| } |
| } |
| |
| return res; |
| } |
| |
| /* wrapper around tmm_clear */ |
| static void clear_pat(struct tmm *tmm, struct tcm_area *area) |
| { |
| struct pat_area p_area = {0}; |
| struct tcm_area slice, area_s; |
| |
| tcm_for_each_slice(slice, *area, area_s) { |
| p_area.x0 = slice.p0.x; |
| p_area.y0 = slice.p0.y; |
| p_area.x1 = slice.p1.x; |
| p_area.y1 = slice.p1.y; |
| |
| tmm_clear(tmm, p_area); |
| } |
| } |
| |
| /* |
| * ID handling methods |
| * ========================================================================== |
| */ |
| |
| /* check if an id is used */ |
| static bool _m_id_in_use(u32 id) |
| { |
| struct mem_info *mi; |
| list_for_each_entry(mi, &blocks, global) |
| if (mi->blk.id == id) |
| return 1; |
| return 0; |
| } |
| |
| /* get an id */ |
| static u32 _m_get_id(void) |
| { |
| static u32 id = 0x2d7ae; |
| |
| /* ensure noone is using this id */ |
| while (_m_id_in_use(id)) { |
| /* generate a new pseudo-random ID */ |
| |
| /* Galois LSFR: 32, 22, 2, 1 */ |
| id = (id >> 1) ^ (u32)((0 - (id & 1u)) & 0x80200003u); |
| } |
| |
| return id; |
| } |
| |
| /* |
| * gid_info handling methods |
| * ========================================================================== |
| */ |
| |
| /* get or create new gid_info object */ |
| static struct gid_info *_m_get_gi(struct process_info *pi, u32 gid) |
| { |
| struct gid_info *gi; |
| |
| /* have mutex */ |
| |
| /* see if group already exist */ |
| list_for_each_entry(gi, &pi->groups, by_pid) { |
| if (gi->gid == gid) |
| goto done; |
| } |
| |
| /* create new group */ |
| gi = kmalloc(sizeof(*gi), GFP_KERNEL); |
| if (!gi) |
| return gi; |
| |
| memset(gi, 0, sizeof(*gi)); |
| INIT_LIST_HEAD(&gi->areas); |
| INIT_LIST_HEAD(&gi->onedim); |
| INIT_LIST_HEAD(&gi->reserved); |
| gi->pi = pi; |
| gi->gid = gid; |
| list_add(&gi->by_pid, &pi->groups); |
| done: |
| /* |
| * Once area is allocated, the group info's ref count will be |
| * decremented as the reference is no longer needed. |
| */ |
| gi->refs++; |
| return gi; |
| } |
| |
| /* free gid_info object if empty */ |
| static void _m_try_free_group(struct gid_info *gi) |
| { |
| /* have mutex */ |
| if (gi && list_empty(&gi->areas) && list_empty(&gi->onedim) && |
| /* also ensure noone is still using this group */ |
| !gi->refs) { |
| BUG_ON(!list_empty(&gi->reserved)); |
| list_del(&gi->by_pid); |
| |
| /* if group is tracking kernel objects, we may free even |
| the process info */ |
| if (gi->pi->kernel && list_empty(&gi->pi->groups)) { |
| list_del(&gi->pi->list); |
| kfree(gi->pi); |
| } |
| |
| kfree(gi); |
| } |
| } |
| |
| /* --- external versions --- */ |
| |
| static struct gid_info *get_gi(struct process_info *pi, u32 gid) |
| { |
| struct gid_info *gi; |
| mutex_lock(&mtx); |
| gi = _m_get_gi(pi, gid); |
| mutex_unlock(&mtx); |
| return gi; |
| } |
| |
| static void release_gi(struct gid_info *gi) |
| { |
| mutex_lock(&mtx); |
| gi->refs--; |
| _m_try_free_group(gi); |
| mutex_unlock(&mtx); |
| } |
| |
| /* |
| * Area handling methods |
| * ========================================================================== |
| */ |
| |
| /* allocate an reserved area of size, alignment and link it to gi */ |
| /* leaves mutex locked to be able to add block to area */ |
| static struct area_info *area_new_m(u16 width, u16 height, u16 align, |
| struct tcm *tcm, struct gid_info *gi) |
| { |
| struct area_info *ai = kmalloc(sizeof(*ai), GFP_KERNEL); |
| if (!ai) |
| return NULL; |
| |
| /* set up empty area info */ |
| memset(ai, 0x0, sizeof(*ai)); |
| INIT_LIST_HEAD(&ai->blocks); |
| |
| /* reserve an allocation area */ |
| if (tcm_reserve_2d(tcm, width, height, align, &ai->area)) { |
| kfree(ai); |
| return NULL; |
| } |
| |
| ai->gi = gi; |
| mutex_lock(&mtx); |
| list_add_tail(&ai->by_gid, &gi->areas); |
| return ai; |
| } |
| |
| /* (must have mutex) free an area */ |
| static inline void _m_area_free(struct area_info *ai) |
| { |
| if (ai) { |
| list_del(&ai->by_gid); |
| kfree(ai); |
| } |
| } |
| |
| static s32 __analize_area(enum tiler_fmt fmt, u32 width, u32 height, |
| u16 *x_area, u16 *y_area, u16 *band, |
| u16 *align, u16 *offs, u16 *in_offs) |
| { |
| /* input: width, height is in pixels, align, offs in bytes */ |
| /* output: x_area, y_area, band, align, offs in slots */ |
| |
| /* slot width, height, and row size */ |
| u32 slot_row, min_align; |
| const struct tiler_geom *g; |
| |
| /* width and height must be positive */ |
| if (!width || !height) |
| return -EINVAL; |
| |
| /* align must be 2 power */ |
| if (*align & (*align - 1)) |
| return -EINVAL; |
| |
| if (fmt == TILFMT_PAGE) { |
| /* adjust size to accomodate offset, only do page alignment */ |
| *align = PAGE_SIZE; |
| *in_offs = *offs & ~PAGE_MASK; |
| width += *in_offs; |
| |
| /* for 1D area keep the height (1), width is in tiler slots */ |
| *x_area = DIV_ROUND_UP(width, tiler.page); |
| *y_area = *band = 1; |
| |
| if (*x_area * *y_area > tiler.width * tiler.height) |
| return -ENOMEM; |
| return 0; |
| } |
| |
| /* format must be valid */ |
| g = tiler.geom(fmt); |
| if (!g) |
| return -EINVAL; |
| |
| /* get the # of bytes per row in 1 slot */ |
| slot_row = g->slot_w * g->bpp; |
| |
| /* how many slots are can be accessed via one physical page */ |
| *band = PAGE_SIZE / slot_row; |
| |
| /* minimum alignment is at least 1 slot. Use default if needed */ |
| min_align = max(slot_row, granularity); |
| *align = ALIGN(*align ? : default_align, min_align); |
| |
| /* align must still be 2 power (in case default_align is wrong) */ |
| if (*align & (*align - 1)) |
| return -EAGAIN; |
| |
| /* offset must be multiple of bpp */ |
| if (*offs & (g->bpp - 1) || *offs >= *align) |
| return -EINVAL; |
| |
| /* round down the offset to the nearest slot size, and increase width |
| to allow space for having the correct offset */ |
| width += (*offs & (min_align - 1)) / g->bpp; |
| if (in_offs) |
| *in_offs = *offs & (min_align - 1); |
| *offs &= ~(min_align - 1); |
| |
| /* expand width to block size */ |
| width = ALIGN(width, min_align / g->bpp); |
| |
| /* adjust to slots */ |
| *x_area = DIV_ROUND_UP(width, g->slot_w); |
| *y_area = DIV_ROUND_UP(height, g->slot_h); |
| *align /= slot_row; |
| *offs /= slot_row; |
| |
| if (*x_area > tiler.width || *y_area > tiler.height) |
| return -ENOMEM; |
| return 0; |
| } |
| |
| /** |
| * Find a place where a 2D block would fit into a 2D area of the |
| * same height. |
| * |
| * @author a0194118 (3/19/2010) |
| * |
| * @param w Width of the block. |
| * @param align Alignment of the block. |
| * @param offs Offset of the block (within alignment) |
| * @param ai Pointer to area info |
| * @param next Pointer to the variable where the next block |
| * will be stored. The block should be inserted |
| * before this block. |
| * |
| * @return the end coordinate (x1 + 1) where a block would fit, |
| * or 0 if it does not fit. |
| * |
| * (must have mutex) |
| */ |
| static u16 _m_blk_find_fit(u16 w, u16 align, u16 offs, |
| struct area_info *ai, struct list_head **before) |
| { |
| int x = ai->area.p0.x + w + offs; |
| struct mem_info *mi; |
| |
| /* area blocks are sorted by x */ |
| list_for_each_entry(mi, &ai->blocks, by_area) { |
| /* check if buffer would fit before this area */ |
| if (x <= mi->area.p0.x) { |
| *before = &mi->by_area; |
| return x; |
| } |
| x = ALIGN(mi->area.p1.x + 1 - offs, align) + w + offs; |
| } |
| *before = &ai->blocks; |
| |
| /* check if buffer would fit after last area */ |
| return (x <= ai->area.p1.x + 1) ? x : 0; |
| } |
| |
| /* (must have mutex) adds a block to an area with certain x coordinates */ |
| static inline |
| struct mem_info *_m_add2area(struct mem_info *mi, struct area_info *ai, |
| u16 x0, u16 w, struct list_head *before) |
| { |
| mi->parent = ai; |
| mi->area = ai->area; |
| mi->area.p0.x = x0; |
| mi->area.p1.x = x0 + w - 1; |
| list_add_tail(&mi->by_area, before); |
| ai->nblocks++; |
| return mi; |
| } |
| |
| static struct mem_info *get_2d_area(u16 w, u16 h, u16 align, u16 offs, u16 band, |
| struct gid_info *gi, struct tcm *tcm) |
| { |
| struct area_info *ai = NULL; |
| struct mem_info *mi = NULL; |
| struct list_head *before = NULL; |
| u16 x = 0; /* this holds the end of a potential area */ |
| |
| /* allocate map info */ |
| |
| /* see if there is available prereserved space */ |
| mutex_lock(&mtx); |
| list_for_each_entry(mi, &gi->reserved, global) { |
| if (mi->area.tcm == tcm && |
| tcm_aheight(mi->area) == h && |
| tcm_awidth(mi->area) == w && |
| (mi->area.p0.x & (align - 1)) == offs) { |
| /* this area is already set up */ |
| |
| /* remove from reserved list */ |
| list_del(&mi->global); |
| goto done; |
| } |
| } |
| mutex_unlock(&mtx); |
| |
| /* if not, reserve a block struct */ |
| mi = kmalloc(sizeof(*mi), GFP_KERNEL); |
| if (!mi) |
| return mi; |
| memset(mi, 0, sizeof(*mi)); |
| |
| /* see if allocation fits in one of the existing areas */ |
| /* this sets x, ai and before */ |
| mutex_lock(&mtx); |
| list_for_each_entry(ai, &gi->areas, by_gid) { |
| if (ai->area.tcm == tcm && |
| tcm_aheight(ai->area) == h) { |
| x = _m_blk_find_fit(w, align, offs, ai, &before); |
| if (x) { |
| _m_add2area(mi, ai, x - w, w, before); |
| goto done; |
| } |
| } |
| } |
| mutex_unlock(&mtx); |
| |
| /* if no area fit, reserve a new one */ |
| ai = area_new_m(ALIGN(w + offs, max(band, align)), h, |
| max(band, align), tcm, gi); |
| if (ai) { |
| _m_add2area(mi, ai, ai->area.p0.x + offs, w, &ai->blocks); |
| } else { |
| /* clean up */ |
| kfree(mi); |
| return NULL; |
| } |
| |
| done: |
| mutex_unlock(&mtx); |
| return mi; |
| } |
| |
| /* layout reserved 2d blocks in a larger area */ |
| /* NOTE: band, w, h, a(lign), o(ffs) is in slots */ |
| static s32 lay_2d(enum tiler_fmt fmt, u16 n, u16 w, u16 h, u16 band, |
| u16 align, u16 offs, struct gid_info *gi, |
| struct list_head *pos) |
| { |
| u16 x, x0, e = ALIGN(w, align), w_res = (n - 1) * e + w; |
| struct mem_info *mi = NULL; |
| struct area_info *ai = NULL; |
| |
| printk(KERN_INFO "packing %u %u buffers into %u width\n", |
| n, w, w_res); |
| |
| /* calculate dimensions, band, offs and alignment in slots */ |
| /* reserve an area */ |
| ai = area_new_m(ALIGN(w_res + offs, max(band, align)), h, |
| max(band, align), tcm[fmt], gi); |
| if (!ai) |
| return -ENOMEM; |
| |
| /* lay out blocks in the reserved area */ |
| for (n = 0, x = offs; x < w_res; x += e, n++) { |
| /* reserve a block struct */ |
| mi = kmalloc(sizeof(*mi), GFP_KERNEL); |
| if (!mi) |
| break; |
| |
| memset(mi, 0, sizeof(*mi)); |
| x0 = ai->area.p0.x + x; |
| _m_add2area(mi, ai, x0, w, &ai->blocks); |
| list_add(&mi->global, pos); |
| } |
| |
| mutex_unlock(&mtx); |
| return n; |
| } |
| |
| /* layout reserved nv12 blocks in a larger area */ |
| /* NOTE: area w(idth), w1 (8-bit block width), h(eight) are in slots */ |
| /* p is a pointer to a packing description, which is a list of offsets in |
| the area for consecutive 8-bit and 16-bit blocks */ |
| static s32 lay_nv12(int n, u16 w, u16 w1, u16 h, struct gid_info *gi, u8 *p) |
| { |
| u16 wh = (w1 + 1) >> 1, width, x0; |
| int m; |
| int a = PAGE_SIZE / tiler.geom(TILFMT_8BIT)->slot_w; |
| |
| struct mem_info *mi = NULL; |
| struct area_info *ai = NULL; |
| struct list_head *pos; |
| |
| /* reserve area */ |
| ai = area_new_m(w, h, a, TILFMT_8BIT, gi); |
| if (!ai) |
| return -ENOMEM; |
| |
| /* lay out blocks in the reserved area */ |
| for (m = 0; m < 2 * n; m++) { |
| width = (m & 1) ? wh : w1; |
| x0 = ai->area.p0.x + *p++; |
| |
| /* get insertion head */ |
| list_for_each(pos, &ai->blocks) { |
| mi = list_entry(pos, struct mem_info, by_area); |
| if (mi->area.p0.x > x0) |
| break; |
| } |
| |
| /* reserve a block struct */ |
| mi = kmalloc(sizeof(*mi), GFP_KERNEL); |
| if (!mi) |
| break; |
| |
| memset(mi, 0, sizeof(*mi)); |
| |
| _m_add2area(mi, ai, x0, width, pos); |
| list_add(&mi->global, &gi->reserved); |
| } |
| |
| mutex_unlock(&mtx); |
| return n; |
| } |
| |
| /* (must have mutex) free block and any freed areas */ |
| static s32 _m_free(struct mem_info *mi) |
| { |
| struct area_info *ai = NULL; |
| struct page *page = NULL; |
| s32 res = 0; |
| u32 i; |
| |
| /* release memory */ |
| if (mi->pg_ptr) { |
| for (i = 0; i < mi->num_pg; i++) { |
| page = (struct page *)mi->pg_ptr[i]; |
| if (page) { |
| if (!PageReserved(page)) |
| SetPageDirty(page); |
| page_cache_release(page); |
| } |
| } |
| kfree(mi->pg_ptr); |
| } else if (mi->mem) { |
| tmm_free(tmm[tiler_fmt(mi->blk.phys)], mi->mem); |
| } |
| clear_pat(tmm[tiler_fmt(mi->blk.phys)], &mi->area); |
| |
| /* safe deletion as list may not have been assigned */ |
| if (mi->global.next) |
| list_del(&mi->global); |
| if (mi->by_area.next) |
| list_del(&mi->by_area); |
| |
| /* remove block from area first if 2D */ |
| if (mi->area.is2d) { |
| ai = mi->parent; |
| |
| /* check to see if area needs removing also */ |
| if (ai && !--ai->nblocks) { |
| res = tcm_free(&ai->area); |
| list_del(&ai->by_gid); |
| /* try to remove parent if it became empty */ |
| _m_try_free_group(ai->gi); |
| kfree(ai); |
| ai = NULL; |
| } |
| } else { |
| /* remove 1D area */ |
| res = tcm_free(&mi->area); |
| /* try to remove parent if it became empty */ |
| _m_try_free_group(mi->parent); |
| } |
| |
| kfree(mi); |
| return res; |
| } |
| |
| /* (must have mutex) returns true if block was freed */ |
| static bool _m_chk_ref(struct mem_info *mi) |
| { |
| /* check references */ |
| if (mi->refs) |
| return 0; |
| |
| if (_m_free(mi)) |
| printk(KERN_ERR "error while removing tiler block\n"); |
| |
| return 1; |
| } |
| |
| /* (must have mutex) */ |
| static inline bool _m_dec_ref(struct mem_info *mi) |
| { |
| if (mi->refs-- <= 1) |
| return _m_chk_ref(mi); |
| |
| return 0; |
| } |
| |
| /* (must have mutex) */ |
| static inline void _m_inc_ref(struct mem_info *mi) |
| { |
| mi->refs++; |
| } |
| |
| /* (must have mutex) returns true if block was freed */ |
| static inline bool _m_try_free(struct mem_info *mi) |
| { |
| if (mi->alloced) { |
| mi->refs--; |
| mi->alloced = false; |
| } |
| return _m_chk_ref(mi); |
| } |
| |
| /* --- external methods --- */ |
| |
| /* find a block by key/id and lock it */ |
| static struct mem_info * |
| find_n_lock(u32 key, u32 id, struct gid_info *gi) { |
| struct area_info *ai = NULL; |
| struct mem_info *mi = NULL; |
| |
| mutex_lock(&mtx); |
| |
| /* if group is not given, look globally */ |
| if (!gi) { |
| list_for_each_entry(mi, &blocks, global) { |
| if (mi->blk.key == key && mi->blk.id == id) |
| goto done; |
| } |
| } else { |
| /* is id is ssptr, we know if block is 1D or 2D by the address, |
| so we optimize lookup */ |
| if (!ssptr_id || |
| tiler_fmt(id) == TILFMT_PAGE) { |
| list_for_each_entry(mi, &gi->onedim, by_area) { |
| if (mi->blk.key == key && mi->blk.id == id) |
| goto done; |
| } |
| } |
| |
| if (!ssptr_id || |
| tiler_fmt(id) != TILFMT_PAGE) { |
| list_for_each_entry(ai, &gi->areas, by_gid) { |
| list_for_each_entry(mi, &ai->blocks, by_area) { |
| if (mi->blk.key == key && |
| mi->blk.id == id) |
| goto done; |
| } |
| } |
| } |
| } |
| |
| mi = NULL; |
| done: |
| /* lock block by increasing its ref count */ |
| if (mi) |
| mi->refs++; |
| |
| mutex_unlock(&mtx); |
| |
| return mi; |
| } |
| |
| /* unlock a block, and optionally free it */ |
| static void unlock_n_free(struct mem_info *mi, bool free) |
| { |
| mutex_lock(&mtx); |
| |
| _m_dec_ref(mi); |
| if (free) |
| _m_try_free(mi); |
| |
| mutex_unlock(&mtx); |
| } |
| |
| /** |
| * Free all blocks in a group: |
| * |
| * allocated blocks, and unreferenced blocks. Any blocks/areas still referenced |
| * will move to the orphaned lists to avoid issues if a new process is created |
| * with the same pid. |
| * |
| * (must have mutex) |
| */ |
| static void destroy_group(struct gid_info *gi) |
| { |
| struct area_info *ai, *ai_; |
| struct mem_info *mi, *mi_; |
| bool ai_autofreed, need2free; |
| |
| mutex_lock(&mtx); |
| |
| /* free all allocated blocks, and remove unreferenced ones */ |
| |
| /* |
| * Group info structs when they become empty on an _m_try_free. |
| * However, if the group info is already empty, we need to |
| * remove it manually |
| */ |
| need2free = list_empty(&gi->areas) && list_empty(&gi->onedim); |
| list_for_each_entry_safe(ai, ai_, &gi->areas, by_gid) { |
| ai_autofreed = true; |
| list_for_each_entry_safe(mi, mi_, &ai->blocks, by_area) |
| ai_autofreed &= _m_try_free(mi); |
| |
| /* save orphaned areas for later removal */ |
| if (!ai_autofreed) { |
| need2free = true; |
| ai->gi = NULL; |
| list_move(&ai->by_gid, &orphan_areas); |
| } |
| } |
| |
| list_for_each_entry_safe(mi, mi_, &gi->onedim, by_area) { |
| if (!_m_try_free(mi)) { |
| need2free = true; |
| /* save orphaned 1D blocks */ |
| mi->parent = NULL; |
| list_move(&mi->by_area, &orphan_onedim); |
| } |
| } |
| |
| /* if group is still alive reserved list should have been |
| emptied as there should be no reference on those blocks */ |
| if (need2free) { |
| BUG_ON(!list_empty(&gi->onedim)); |
| BUG_ON(!list_empty(&gi->areas)); |
| _m_try_free_group(gi); |
| } |
| |
| mutex_unlock(&mtx); |
| } |
| |
| /* release (reserved) blocks */ |
| static void release_blocks(struct list_head *reserved) |
| { |
| struct mem_info *mi, *mi_; |
| |
| mutex_lock(&mtx); |
| |
| /* find block in global list and free it */ |
| list_for_each_entry_safe(mi, mi_, reserved, global) { |
| BUG_ON(mi->refs || mi->alloced); |
| _m_free(mi); |
| } |
| mutex_unlock(&mtx); |
| } |
| |
| /* add reserved blocks to a group */ |
| static void add_reserved_blocks(struct list_head *reserved, struct gid_info *gi) |
| { |
| mutex_lock(&mtx); |
| list_splice_init(reserved, &gi->reserved); |
| mutex_unlock(&mtx); |
| } |
| |
| /* find a block by ssptr */ |
| static struct mem_info *find_block_by_ssptr(u32 sys_addr) |
| { |
| struct mem_info *i; |
| struct tcm_pt pt; |
| u32 x, y; |
| enum tiler_fmt fmt; |
| const struct tiler_geom *g; |
| |
| fmt = tiler_fmt(sys_addr); |
| if (fmt == TILFMT_INVALID) |
| return NULL; |
| |
| g = tiler.geom(fmt); |
| |
| /* convert x & y pixel coordinates to slot coordinates */ |
| tiler.xy(sys_addr, &x, &y); |
| pt.x = x / g->slot_w; |
| pt.y = y / g->slot_h; |
| |
| mutex_lock(&mtx); |
| list_for_each_entry(i, &blocks, global) { |
| if (tiler_fmt(i->blk.phys) == tiler_fmt(sys_addr) && |
| tcm_is_in(pt, i->area)) { |
| i->refs++; |
| goto found; |
| } |
| } |
| i = NULL; |
| |
| found: |
| mutex_unlock(&mtx); |
| return i; |
| } |
| |
| /* find a block by ssptr */ |
| static void fill_block_info(struct mem_info *i, struct tiler_block_info *blk) |
| { |
| blk->fmt = tiler_fmt(i->blk.phys); |
| #ifdef CONFIG_TILER_EXPOSE_SSPTR |
| blk->ssptr = i->blk.phys; |
| #endif |
| if (blk->fmt == TILFMT_PAGE) { |
| blk->dim.len = i->blk.width; |
| blk->group_id = ((struct gid_info *) i->parent)->gid; |
| } else { |
| blk->stride = tiler_vstride(&i->blk); |
| blk->dim.area.width = i->blk.width; |
| blk->dim.area.height = i->blk.height; |
| blk->group_id = ((struct area_info *) i->parent)->gi->gid; |
| } |
| blk->id = i->blk.id; |
| blk->key = i->blk.key; |
| blk->offs = i->blk.phys & ~PAGE_MASK; |
| blk->align = PAGE_SIZE; |
| } |
| |
| /* |
| * Block operations |
| * ========================================================================== |
| */ |
| |
| static struct mem_info *__get_area(enum tiler_fmt fmt, u32 width, u32 height, |
| u16 align, u16 offs, struct gid_info *gi) |
| { |
| u16 x, y, band, in_offs = 0; |
| struct mem_info *mi = NULL; |
| const struct tiler_geom *g = tiler.geom(fmt); |
| |
| /* calculate dimensions, band, offs and alignment in slots */ |
| if (__analize_area(fmt, width, height, &x, &y, &band, &align, &offs, |
| &in_offs)) |
| return NULL; |
| |
| if (fmt == TILFMT_PAGE) { |
| /* 1D areas don't pack */ |
| mi = kmalloc(sizeof(*mi), GFP_KERNEL); |
| if (!mi) |
| return NULL; |
| memset(mi, 0x0, sizeof(*mi)); |
| |
| if (tcm_reserve_1d(tcm[fmt], x * y, &mi->area)) { |
| kfree(mi); |
| return NULL; |
| } |
| |
| mutex_lock(&mtx); |
| mi->parent = gi; |
| list_add(&mi->by_area, &gi->onedim); |
| } else { |
| mi = get_2d_area(x, y, align, offs, band, gi, tcm[fmt]); |
| if (!mi) |
| return NULL; |
| |
| mutex_lock(&mtx); |
| } |
| |
| list_add(&mi->global, &blocks); |
| mi->alloced = true; |
| mi->refs++; |
| gi->refs--; |
| mutex_unlock(&mtx); |
| |
| mi->blk.phys = tiler.addr(fmt, |
| mi->area.p0.x * g->slot_w, mi->area.p0.y * g->slot_h) |
| + in_offs; |
| return mi; |
| } |
| |
| static s32 alloc_block(enum tiler_fmt fmt, u32 width, u32 height, |
| u32 align, u32 offs, u32 key, u32 gid, struct process_info *pi, |
| struct mem_info **info) |
| { |
| struct mem_info *mi = NULL; |
| struct gid_info *gi = NULL; |
| |
| *info = NULL; |
| |
| /* only support up to page alignment */ |
| if (align > PAGE_SIZE || offs >= (align ? : default_align) || !pi) |
| return -EINVAL; |
| |
| /* get group context */ |
| mutex_lock(&mtx); |
| gi = _m_get_gi(pi, gid); |
| mutex_unlock(&mtx); |
| |
| if (!gi) |
| return -ENOMEM; |
| |
| /* reserve area in tiler container */ |
| mi = __get_area(fmt, width, height, align, offs, gi); |
| if (!mi) { |
| mutex_lock(&mtx); |
| gi->refs--; |
| _m_try_free_group(gi); |
| mutex_unlock(&mtx); |
| return -ENOMEM; |
| } |
| |
| mi->blk.width = width; |
| mi->blk.height = height; |
| mi->blk.key = key; |
| if (ssptr_id) { |
| mi->blk.id = mi->blk.phys; |
| } else { |
| mutex_lock(&mtx); |
| mi->blk.id = _m_get_id(); |
| mutex_unlock(&mtx); |
| } |
| |
| /* allocate and map if mapping is supported */ |
| if (tmm_can_map(tmm[fmt])) { |
| mi->num_pg = tcm_sizeof(mi->area); |
| |
| mi->mem = tmm_get(tmm[fmt], mi->num_pg); |
| if (!mi->mem) |
| goto cleanup; |
| |
| /* Ensure the data reaches to main memory before PAT refill */ |
| wmb(); |
| |
| /* program PAT */ |
| if (refill_pat(tmm[fmt], &mi->area, mi->mem)) |
| goto cleanup; |
| } |
| *info = mi; |
| return 0; |
| |
| cleanup: |
| mutex_lock(&mtx); |
| _m_free(mi); |
| mutex_unlock(&mtx); |
| return -ENOMEM; |
| |
| } |
| |
| static s32 map_block(enum tiler_fmt fmt, u32 width, u32 height, |
| u32 key, u32 gid, struct process_info *pi, |
| struct mem_info **info, u32 usr_addr) |
| { |
| u32 i = 0, tmp = -1, *mem = NULL; |
| u8 write = 0; |
| s32 res = -ENOMEM; |
| struct mem_info *mi = NULL; |
| struct page *page = NULL; |
| struct task_struct *curr_task = current; |
| struct mm_struct *mm = current->mm; |
| struct vm_area_struct *vma = NULL; |
| struct gid_info *gi = NULL; |
| |
| *info = NULL; |
| |
| /* we only support mapping a user buffer in page mode */ |
| if (fmt != TILFMT_PAGE) |
| return -EPERM; |
| |
| /* check if mapping is supported by tmm */ |
| if (!tmm_can_map(tmm[fmt])) |
| return -EPERM; |
| |
| /* get group context */ |
| mutex_lock(&mtx); |
| gi = _m_get_gi(pi, gid); |
| mutex_unlock(&mtx); |
| |
| if (!gi) |
| return -ENOMEM; |
| |
| /* reserve area in tiler container */ |
| mi = __get_area(fmt, width, height, 0, 0, gi); |
| if (!mi) { |
| mutex_lock(&mtx); |
| gi->refs--; |
| _m_try_free_group(gi); |
| mutex_unlock(&mtx); |
| return -ENOMEM; |
| } |
| |
| mi->blk.width = width; |
| mi->blk.height = height; |
| mi->blk.key = key; |
| if (ssptr_id) { |
| mi->blk.id = mi->blk.phys; |
| } else { |
| mutex_lock(&mtx); |
| mi->blk.id = _m_get_id(); |
| mutex_unlock(&mtx); |
| } |
| |
| mi->usr = usr_addr; |
| |
| /* allocate pages */ |
| mi->num_pg = tcm_sizeof(mi->area); |
| |
| mem = kmalloc(mi->num_pg * sizeof(*mem), GFP_KERNEL); |
| if (!mem) |
| goto done; |
| memset(mem, 0x0, sizeof(*mem) * mi->num_pg); |
| |
| mi->pg_ptr = kmalloc(mi->num_pg * sizeof(*mi->pg_ptr), GFP_KERNEL); |
| if (!mi->pg_ptr) |
| goto done; |
| memset(mi->pg_ptr, 0x0, sizeof(*mi->pg_ptr) * mi->num_pg); |
| |
| /* |
| * Important Note: usr_addr is mapped from user |
| * application process to current process - it must lie |
| * completely within the current virtual memory address |
| * space in order to be of use to us here. |
| */ |
| down_read(&mm->mmap_sem); |
| vma = find_vma(mm, mi->usr); |
| res = -EFAULT; |
| |
| /* |
| * It is observed that under some circumstances, the user |
| * buffer is spread across several vmas, so loop through |
| * and check if the entire user buffer is covered. |
| */ |
| while ((vma) && (mi->usr + width > vma->vm_end)) { |
| /* jump to the next VMA region */ |
| vma = find_vma(mm, vma->vm_end + 1); |
| } |
| if (!vma) { |
| printk(KERN_ERR "Failed to get the vma region for " |
| "user buffer.\n"); |
| goto fault; |
| } |
| |
| if (vma->vm_flags & (VM_WRITE | VM_MAYWRITE)) |
| write = 1; |
| |
| tmp = mi->usr; |
| for (i = 0; i < mi->num_pg; i++) { |
| if (get_user_pages(curr_task, mm, tmp, 1, write, 1, &page, |
| NULL)) { |
| if (page_count(page) < 1) { |
| printk(KERN_ERR "Bad page count from" |
| "get_user_pages()\n"); |
| } |
| mi->pg_ptr[i] = (u32)page; |
| mem[i] = page_to_phys(page); |
| tmp += PAGE_SIZE; |
| } else { |
| printk(KERN_ERR "get_user_pages() failed\n"); |
| goto fault; |
| } |
| } |
| up_read(&mm->mmap_sem); |
| |
| /* Ensure the data reaches to main memory before PAT refill */ |
| wmb(); |
| |
| if (refill_pat(tmm[fmt], &mi->area, mem)) |
| goto fault; |
| |
| res = 0; |
| *info = mi; |
| goto done; |
| fault: |
| up_read(&mm->mmap_sem); |
| done: |
| if (res) { |
| mutex_lock(&mtx); |
| _m_free(mi); |
| mutex_unlock(&mtx); |
| } |
| kfree(mem); |
| return res; |
| } |
| |
| /* |
| * Driver code |
| * ========================================================================== |
| */ |
| |
| static s32 __init tiler_init(void) |
| { |
| dev_t dev = 0; |
| s32 r = -1; |
| struct device *device = NULL; |
| struct tcm_pt div_pt; |
| struct tcm *sita = NULL; |
| struct tmm *tmm_pat = NULL; |
| |
| tiler.alloc = alloc_block; |
| tiler.map = map_block; |
| tiler.lock = find_n_lock; |
| tiler.unlock_free = unlock_n_free; |
| tiler.lay_2d = lay_2d; |
| tiler.lay_nv12 = lay_nv12; |
| tiler.destroy_group = destroy_group; |
| tiler.lock_by_ssptr = find_block_by_ssptr; |
| tiler.describe = fill_block_info; |
| tiler.get_gi = get_gi; |
| tiler.release_gi = release_gi; |
| tiler.release = release_blocks; |
| tiler.add_reserved = add_reserved_blocks; |
| tiler.analize = __analize_area; |
| tiler_geom_init(&tiler); |
| tiler_reserve_init(&tiler); |
| tiler_iface_init(&tiler); |
| |
| /* check module parameters for correctness */ |
| if (default_align > PAGE_SIZE || |
| default_align & (default_align - 1) || |
| granularity < 1 || granularity > PAGE_SIZE || |
| granularity & (granularity - 1)) |
| return -EINVAL; |
| |
| /* |
| * Array of physical pages for PAT programming, which must be a 16-byte |
| * aligned physical address. |
| */ |
| dmac_va = dma_alloc_coherent(NULL, tiler.width * tiler.height * |
| sizeof(*dmac_va), &dmac_pa, GFP_ATOMIC); |
| if (!dmac_va) |
| return -ENOMEM; |
| |
| /* Allocate tiler container manager (we share 1 on OMAP4) */ |
| div_pt.x = tiler.width; /* hardcoded default */ |
| div_pt.y = (3 * tiler.height) / 4; |
| sita = sita_init(tiler.width, tiler.height, (void *)&div_pt); |
| |
| tcm[TILFMT_8BIT] = sita; |
| tcm[TILFMT_16BIT] = sita; |
| tcm[TILFMT_32BIT] = sita; |
| tcm[TILFMT_PAGE] = sita; |
| |
| /* Allocate tiler memory manager (must have 1 unique TMM per TCM ) */ |
| tmm_pat = tmm_pat_init(0); |
| tmm[TILFMT_8BIT] = tmm_pat; |
| tmm[TILFMT_16BIT] = tmm_pat; |
| tmm[TILFMT_32BIT] = tmm_pat; |
| tmm[TILFMT_PAGE] = tmm_pat; |
| |
| tiler.nv12_packed = tcm[TILFMT_8BIT] == tcm[TILFMT_16BIT]; |
| |
| tiler_device = kmalloc(sizeof(*tiler_device), GFP_KERNEL); |
| if (!tiler_device || !sita || !tmm_pat) { |
| r = -ENOMEM; |
| goto error; |
| } |
| |
| memset(tiler_device, 0x0, sizeof(*tiler_device)); |
| if (tiler_major) { |
| dev = MKDEV(tiler_major, tiler_minor); |
| r = register_chrdev_region(dev, 1, "tiler"); |
| } else { |
| r = alloc_chrdev_region(&dev, tiler_minor, 1, "tiler"); |
| tiler_major = MAJOR(dev); |
| } |
| |
| cdev_init(&tiler_device->cdev, tiler.fops); |
| tiler_device->cdev.owner = THIS_MODULE; |
| tiler_device->cdev.ops = tiler.fops; |
| |
| r = cdev_add(&tiler_device->cdev, dev, 1); |
| if (r) |
| printk(KERN_ERR "cdev_add():failed\n"); |
| |
| tilerdev_class = class_create(THIS_MODULE, "tiler"); |
| |
| if (IS_ERR(tilerdev_class)) { |
| printk(KERN_ERR "class_create():failed\n"); |
| goto error; |
| } |
| |
| device = device_create(tilerdev_class, NULL, dev, NULL, "tiler"); |
| if (device == NULL) |
| printk(KERN_ERR "device_create() fail\n"); |
| |
| r = platform_driver_register(&tiler_driver_ldm); |
| |
| mutex_init(&mtx); |
| INIT_LIST_HEAD(&blocks); |
| INIT_LIST_HEAD(&orphan_areas); |
| INIT_LIST_HEAD(&orphan_onedim); |
| |
| error: |
| /* TODO: error handling for device registration */ |
| if (r) { |
| kfree(tiler_device); |
| tcm_deinit(sita); |
| tmm_deinit(tmm_pat); |
| dma_free_coherent(NULL, tiler.width * tiler.height * |
| sizeof(*dmac_va), dmac_va, dmac_pa); |
| } |
| |
| return r; |
| } |
| |
| static void __exit tiler_exit(void) |
| { |
| int i, j; |
| |
| mutex_lock(&mtx); |
| |
| /* free all process data */ |
| tiler.cleanup(); |
| |
| /* all lists should have cleared */ |
| BUG_ON(!list_empty(&blocks)); |
| BUG_ON(!list_empty(&orphan_onedim)); |
| BUG_ON(!list_empty(&orphan_areas)); |
| |
| mutex_unlock(&mtx); |
| |
| dma_free_coherent(NULL, tiler.width * tiler.height * sizeof(*dmac_va), |
| dmac_va, dmac_pa); |
| |
| /* close containers only once */ |
| for (i = TILFMT_MIN; i <= TILFMT_MAX; i++) { |
| /* remove identical containers (tmm is unique per tcm) */ |
| for (j = i + 1; j <= TILFMT_MAX; j++) |
| if (tcm[i] == tcm[j]) { |
| tcm[j] = NULL; |
| tmm[j] = NULL; |
| } |
| |
| tcm_deinit(tcm[i]); |
| tmm_deinit(tmm[i]); |
| } |
| |
| mutex_destroy(&mtx); |
| platform_driver_unregister(&tiler_driver_ldm); |
| cdev_del(&tiler_device->cdev); |
| kfree(tiler_device); |
| device_destroy(tilerdev_class, MKDEV(tiler_major, tiler_minor)); |
| class_destroy(tilerdev_class); |
| } |
| |
| MODULE_LICENSE("GPL v2"); |
| MODULE_AUTHOR("Lajos Molnar <molnar@ti.com>"); |
| MODULE_AUTHOR("David Sin <davidsin@ti.com>"); |
| module_init(tiler_init); |
| module_exit(tiler_exit); |