utgard/r5p1/linux/mali_memory_manager.c - manifest_repos/mali-driver - Git at Google

 /*
  * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
  *
  * A copy of the licence is included with the program, and can also be obtained from Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */

 #include <linux/list.h>
 #include <linux/mm.h>
 #include <linux/mm_types.h>
 #include <linux/fs.h>
 #include <linux/dma-mapping.h>
 #include <linux/slab.h>
 #include <linux/version.h>
 #include <linux/platform_device.h>
 #if defined(CONFIG_DMA_SHARED_BUFFER)
 #include <linux/dma-buf.h>
 #endif
 #include <linux/idr.h>

 #include "mali_osk.h"
 #include "mali_osk_mali.h"
 #include "mali_kernel_linux.h"
 #include "mali_scheduler.h"
 #include "mali_memory.h"
 #include "mali_memory_os_alloc.h"
 #if defined(CONFIG_DMA_SHARED_BUFFER)
 #include "mali_memory_dma_buf.h"
 #endif
 #if defined(CONFIG_MALI400_UMP)
 #include "mali_memory_ump.h"
 #endif
 #include "mali_memory_manager.h"
 #include "mali_memory_virtual.h"
 #include "mali_memory_util.h"
 #include "mali_memory_external.h"
 #include "mali_memory_block_alloc.h"

 #define MALI_S32_MAX 0x7fffffff

 /*
 * New memory system interface
 */

 /*inti idr for backend memory */
 struct idr mali_backend_idr;
 struct mutex mali_idr_mutex;

 /* init allocation manager */
 int mali_memory_manager_init(struct mali_allocation_manager *mgr)
 {
 	/* init Locks */
 	rwlock_init(&mgr->vm_lock);
 	mutex_init(&mgr->list_mutex);

 	/* init link */
 	INIT_LIST_HEAD(&mgr->head);

 	/* init RB tree */
 	mgr->allocation_mgr_rb = RB_ROOT;
 	mgr->mali_allocation_nr = 0;
 	return 0;
 }

 /* deinit allocation manager
 * Do some check for debug
 */
 void mali_memory_manager_uninit(struct mali_allocation_manager *mgr)
 {
 	/* check RB tree is empty */
 	MALI_DEBUG_ASSERT(((void *)(mgr->allocation_mgr_rb.rb_node) == (void *)rb_last(&mgr->allocation_mgr_rb)));
 	/* check allocation List */
 	MALI_DEBUG_ASSERT(list_empty(&mgr->head));
 }

 /* Prepare memory descriptor */
 static mali_mem_allocation *mali_mem_allocation_struct_create(struct mali_session_data *session)
 {
 	mali_mem_allocation *mali_allocation;

 	/* Allocate memory */
 	mali_allocation = (mali_mem_allocation *)kzalloc(sizeof(mali_mem_allocation), GFP_KERNEL);
 	if (NULL == mali_allocation) {
 		MALI_DEBUG_PRINT(1, ("mali_mem_allocation_struct_create: descriptor was NULL\n"));
 		return NULL;
 	}

 	MALI_DEBUG_CODE(mali_allocation->magic = MALI_MEM_ALLOCATION_VALID_MAGIC);

 	/* do init */
 	mali_allocation->flags = 0;
 	mali_allocation->session = session;

 	INIT_LIST_HEAD(&mali_allocation->list);
 	kref_init(&mali_allocation->ref);

 	/**
 	*add to session list
 	*/
 	mutex_lock(&session->allocation_mgr.list_mutex);
 	list_add_tail(&mali_allocation->list, &session->allocation_mgr.head);
 	session->allocation_mgr.mali_allocation_nr++;
 	mutex_unlock(&session->allocation_mgr.list_mutex);

 	return mali_allocation;
 }


 void  mali_mem_allocation_struct_destory(mali_mem_allocation *alloc)
 {
 	MALI_DEBUG_ASSERT_POINTER(alloc);
 	MALI_DEBUG_ASSERT_POINTER(alloc->session);
 	mutex_lock(&alloc->session->allocation_mgr.list_mutex);
 	list_del(&alloc->list);
 	alloc->session->allocation_mgr.mali_allocation_nr--;
 	mutex_unlock(&alloc->session->allocation_mgr.list_mutex);

 	kfree(alloc);
 }

 int mali_mem_backend_struct_create(mali_mem_backend **backend, u32 psize)
 {
 	mali_mem_backend *mem_backend = NULL;
 	s32 ret = -ENOSPC;
 	s32 index = -1;
 	*backend = (mali_mem_backend *)kzalloc(sizeof(mali_mem_backend), GFP_KERNEL);
 	if (NULL == *backend) {
 		MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: backend descriptor was NULL\n"));
 		return -1;
 	}
 	mem_backend = *backend;
 	mem_backend->size = psize;
 	/* link backend with id */
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
 again:
 	if (!idr_pre_get(&mali_backend_idr, GFP_KERNEL)) {
 		kfree(mem_backend);
 		return -ENOMEM;
 	}
 	mutex_lock(&mali_idr_mutex);
 	ret = idr_get_new_above(&mali_backend_idr, mem_backend, 1, &index);
 	mutex_unlock(&mali_idr_mutex);

 	if (-ENOSPC == ret) {
 		kfree(mem_backend);
 		return -ENOSPC;
 	}
 	if (-EAGAIN == ret)
 		goto again;
 #else
 	mutex_lock(&mali_idr_mutex);
 	ret = idr_alloc(&mali_backend_idr, mem_backend, 1, MALI_S32_MAX, GFP_KERNEL);
 	mutex_unlock(&mali_idr_mutex);
 	index = ret;
 	if (ret < 0) {
 		MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: Can't allocate idr for backend! \n"));
 		kfree(mem_backend);
 		return -ENOSPC;
 	}
 #endif
 	return index;
 }

 static void mali_mem_backend_struct_destory(mali_mem_backend **backend, s32 backend_handle)
 {
 	mali_mem_backend *mem_backend = *backend;

 	mutex_lock(&mali_idr_mutex);
 	idr_remove(&mali_backend_idr, backend_handle);
 	mutex_unlock(&mali_idr_mutex);
 	kfree(mem_backend);
 	*backend = NULL;
 }


 /* Set GPU MMU properties */
 static void _mali_memory_gpu_map_property_set(u32 *properties, u32 flags)
 {

 	if (_MALI_MEMORY_GPU_READ_ALLOCATE & flags) {
 		*properties = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE;
 	} else {
 		*properties = MALI_MMU_FLAGS_DEFAULT;
 	}
 }


 /**
 *  function@_mali_ukk_mem_allocate - allocate mali memory
 */
 _mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args)
 {
 	struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
 	mali_mem_backend *mem_backend = NULL;
 	_mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
 	int retval = 0;
 	mali_mem_allocation *mali_allocation = NULL;
 	struct mali_vma_node *mali_vma_node = NULL;

 	MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_allocate, vaddr=0x%x, size =0x%x! \n", args->gpu_vaddr, args->psize));

 	/* Check if the address is allocated
 	*  Can we trust User mode?
 	*/
 	mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->gpu_vaddr, 0);
 	if (unlikely(mali_vma_node)) {
 		/* Not support yet */
 		MALI_DEBUG_ASSERT(0);
 		return _MALI_OSK_ERR_FAULT;
 	}

 	/**
 	*create mali memory allocation
 	*/
 	mali_allocation = mali_mem_allocation_struct_create(session);

 	if (mali_allocation == NULL) {
 		MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_allocate: Failed to create allocation struct! \n"));
 		return _MALI_OSK_ERR_NOMEM;
 	}
 	mali_allocation->psize = args->psize;
 	mali_allocation->vsize = args->vsize;

 	/* check if have dedicated memory */
 	if (MALI_TRUE == mali_memory_have_dedicated_memory()) {
 		mali_allocation->type = MALI_MEM_BLOCK;
 	} else {
 		mali_allocation->type = MALI_MEM_OS;
 	}

 	/**
 	*add allocation node to RB tree for index
 	*/
 	mali_allocation->mali_vma_node.vm_node.start = args->gpu_vaddr;
 	mali_allocation->mali_vma_node.vm_node.size = args->vsize;

 	mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node);

 	/* check if need to allocate backend */
 	if (mali_allocation->psize == 0)
 		return _MALI_OSK_ERR_OK;

 	/**
 	*allocate physical backend & pages
 	*/
 	if (likely(mali_allocation->psize > 0)) {
 		mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, args->psize);
 		if (mali_allocation->backend_handle < 0) {
 			ret = _MALI_OSK_ERR_NOMEM;
 			MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n"));
 			goto failed_alloc_backend;
 		}

 		mem_backend->mali_allocation = mali_allocation;
 		mem_backend->type = mali_allocation->type;

 		if (mem_backend->type == MALI_MEM_OS) {
 			retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size);
 		} else if (mem_backend->type == MALI_MEM_BLOCK) {
 			/* try to allocated from BLOCK memory first, then try OS memory if failed.*/
 			if (mali_mem_block_alloc(&mem_backend->block_mem, mem_backend->size)) {
 				retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size);
 				mem_backend->type = MALI_MEM_OS;
 				mali_allocation->type = MALI_MEM_OS;
 			}
 		} else {
 			/* ONLY support mem_os type */
 			MALI_DEBUG_ASSERT(0);
 		}

 		if (retval) {
 			ret = _MALI_OSK_ERR_NOMEM;
 			MALI_DEBUG_PRINT(1, (" can't allocate enough pages! \n"));
 			goto failed_alloc_pages;
 		}
 	}

 	/**
 	*map to GPU side
 	*/
 	mali_allocation->mali_mapping.addr = args->gpu_vaddr;

 	/* set gpu mmu propery */
 	_mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags);

 	if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) {
 		_mali_osk_mutex_wait(session->memory_lock);
 		/* Map on Mali */
 		ret = mali_mem_mali_map_prepare(mali_allocation);
 		if (0 != ret) {
 			MALI_DEBUG_PRINT(1, (" prepare map fail! \n"));
 			goto failed_gpu_map;
 		}
 		/* only support os memory type now */
 		if (mem_backend->type == MALI_MEM_OS) {
 			mali_mem_os_mali_map(mem_backend, args->gpu_vaddr,
 					     mali_allocation->mali_mapping.properties);
 		} else if (mem_backend->type == MALI_MEM_BLOCK) {
 			mali_mem_block_mali_map(&mem_backend->block_mem, session, args->gpu_vaddr,
 						mali_allocation->mali_mapping.properties);
 		} else {
 			/* Not support yet */
 			MALI_DEBUG_ASSERT(0);
 		}
 		session->mali_mem_array[mem_backend->type] += mem_backend->size;
 		if (session->mali_mem_array[MALI_MEM_OS] + session->mali_mem_array[MALI_MEM_BLOCK] > session->max_mali_mem_allocated) {
 			session->max_mali_mem_allocated = session->mali_mem_array[MALI_MEM_OS] + session->mali_mem_array[MALI_MEM_BLOCK];
 		}
 		_mali_osk_mutex_signal(session->memory_lock);
 	}

 	return _MALI_OSK_ERR_OK;

 failed_gpu_map:
 	_mali_osk_mutex_signal(session->memory_lock);
 	if (mem_backend->type == MALI_MEM_OS) {
 		mali_mem_os_free(&mem_backend->os_mem);
 	} else {
 		mali_mem_block_free(&mem_backend->block_mem);
 	}
 failed_alloc_pages:
 	mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle);
 failed_alloc_backend:

 	mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node);
 	mali_mem_allocation_struct_destory(mali_allocation);

 	return ret;
 }


 _mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args)
 {
 	struct  mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
 	u32 vaddr = args->gpu_vaddr;
 	mali_mem_allocation *mali_alloc = NULL;
 	struct mali_vma_node *mali_vma_node = NULL;

 	/* find mali allocation structure by vaddress*/
 	mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, vaddr, 0);

 	MALI_DEBUG_ASSERT(NULL != mali_vma_node);
 	mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);

 	if (mali_alloc)
 		/* check ref_count */
 		mali_allocation_unref(&mali_alloc);

 	return _MALI_OSK_ERR_OK;
 }


 /**
 * Function _mali_ukk_mem_bind -- bind a external memory to a new GPU address
 * It will allocate a new mem allocation and bind external memory to it.
 * Supported backend type are:
 * _MALI_MEMORY_BIND_BACKEND_UMP
 * _MALI_MEMORY_BIND_BACKEND_DMA_BUF
 * _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY
 * CPU access is not supported yet
 */
 _mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args)
 {
 	struct  mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
 	mali_mem_backend *mem_backend = NULL;
 	_mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
 	mali_mem_allocation *mali_allocation = NULL;
 	MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_bind, vaddr=0x%x, size =0x%x! \n", args->vaddr, args->size));

 	/**
 	* allocate mali allocation.
 	*/
 	mali_allocation = mali_mem_allocation_struct_create(session);

 	if (mali_allocation == NULL) {
 		return _MALI_OSK_ERR_NOMEM;
 	}
 	mali_allocation->psize = args->size;
 	mali_allocation->vsize = args->size;
 	mali_allocation->mali_mapping.addr = args->vaddr;

 	/* add allocation node to RB tree for index  */
 	mali_allocation->mali_vma_node.vm_node.start = args->vaddr;
 	mali_allocation->mali_vma_node.vm_node.size = args->size;
 	mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node);

 	/* allocate backend*/
 	if (mali_allocation->psize > 0) {
 		mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize);
 		if (mali_allocation->backend_handle < 0) {
 			goto Failed_alloc_backend;
 		}

 	} else {
 		goto Failed_alloc_backend;
 	}

 	mem_backend->size = mali_allocation->psize;
 	mem_backend->mali_allocation = mali_allocation;

 	switch (args->flags & _MALI_MEMORY_BIND_BACKEND_MASK) {
 	case  _MALI_MEMORY_BIND_BACKEND_UMP:
 #if defined(CONFIG_MALI400_UMP)
 		mali_allocation->type = MALI_MEM_UMP;
 		mem_backend->type = MALI_MEM_UMP;
 		ret = mali_memory_bind_ump_buf(mali_allocation, mem_backend,
 					       args->mem_union.bind_ump.secure_id, args->mem_union.bind_ump.flags);
 		if (_MALI_OSK_ERR_OK != ret) {
 			MALI_DEBUG_PRINT(1, ("Bind ump buf failed\n"));
 			goto  Failed_bind_backend;
 		}
 #else
 		MALI_DEBUG_PRINT(1, ("UMP not supported\n"));
 		goto Failed_bind_backend;
 #endif
 		break;
 	case  _MALI_MEMORY_BIND_BACKEND_DMA_BUF:
 #if defined(CONFIG_DMA_SHARED_BUFFER)
 		mali_allocation->type = MALI_MEM_DMA_BUF;
 		mem_backend->type = MALI_MEM_DMA_BUF;
 		ret = mali_memory_bind_dma_buf(mali_allocation, mem_backend,
 					       args->mem_union.bind_dma_buf.mem_fd, args->mem_union.bind_dma_buf.flags);
 		if (_MALI_OSK_ERR_OK != ret) {
 			MALI_DEBUG_PRINT(1, ("Bind dma buf failed\n"));
 			goto Failed_bind_backend;
 		}
 #else
 		MALI_DEBUG_PRINT(1, ("DMA not supported\n"));
 		goto Failed_bind_backend;
 #endif
 		break;
 	case _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY:
 		/* not allowed */
 		MALI_DEBUG_ASSERT(0);
 		break;

 	case _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY:
 		mali_allocation->type = MALI_MEM_EXTERNAL;
 		mem_backend->type = MALI_MEM_EXTERNAL;
 		ret = mali_memory_bind_ext_mem(mali_allocation, mem_backend, args->mem_union.bind_ext_memory.phys_addr,
 					       args->mem_union.bind_ext_memory.flags);
 		if (_MALI_OSK_ERR_OK != ret) {
 			MALI_DEBUG_PRINT(1, ("Bind external buf failed\n"));
 			goto Failed_bind_backend;
 		}
 		break;

 	case _MALI_MEMORY_BIND_BACKEND_EXT_COW:
 		/* not allowed */
 		MALI_DEBUG_ASSERT(0);
 		break;

 	default:
 		MALI_DEBUG_ASSERT(0);
 		break;
 	}
 	return _MALI_OSK_ERR_OK;


 Failed_bind_backend:
 	mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle);

 Failed_alloc_backend:
 	mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node);
 	mali_mem_allocation_struct_destory(mali_allocation);

 	MALI_DEBUG_PRINT(1, (" _mali_ukk_mem_bind, return ERROR! \n"));
 	return ret;
 }


 /*
 * Function _mali_ukk_mem_unbind -- unbind a external memory to a new GPU address
 * This function unbind the backend memory and free the allocation
 * no ref_count for this type of memory
 */
 _mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args)
 {
 	/**/
 	struct  mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
 	mali_mem_allocation *mali_allocation = NULL;
 	struct mali_vma_node *mali_vma_node = NULL;
 	u32 mali_addr = args->vaddr;
 	MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_unbind, vaddr=0x%x! \n", args->vaddr));

 	/* find the allocation by vaddr */
 	mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
 	if (likely(mali_vma_node)) {
 		MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start);
 		mali_allocation = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
 	} else {
 		MALI_DEBUG_ASSERT(NULL != mali_vma_node);
 		/* Not support yet */
 		MALI_DEBUG_ASSERT(0);
 		return _MALI_OSK_ERR_INVALID_ARGS;
 	}

 	if (NULL != mali_allocation)
 		/* check ref_count */
 		mali_allocation_unref(&mali_allocation);
 	return _MALI_OSK_ERR_OK;
 }


 /*
 * Function _mali_ukk_mem_cow --  COW for an allocation
 * This function allocate new pages for  a range (range, range+size) of allocation
 *  And Map it(keep use the not in range pages from target allocation ) to an GPU vaddr
 */
 _mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args)
 {
 	_mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;

 	/* create new alloction if needed */

 	/* Get the target allocation and it's backend*/

 	/* allocate new pages from os mem for modified range */


 	/* fill the COW backend, all pages for this allocation
 	*  including the new page for modified range and pages not modified in old allocation.
 	* Do Add ref to pages from target allocation
 	*/


 	/* map it to GPU side */
 	return ret;
 }

 /**
 *  attach a backend to an exist mali allocation
 */


 /**
 *  deattach a backend from an exist mali allocation
 */
	/*
	* Copyright (C) 2013-2015 ARM Limited. All rights reserved.
	*
	* This program is free software and is provided to you under the terms of the GNU General Public License version 2
	* as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
	*
	* A copy of the licence is included with the program, and can also be obtained from Free Software
	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
	*/

	#include <linux/list.h>
	#include <linux/mm.h>
	#include <linux/mm_types.h>
	#include <linux/fs.h>
	#include <linux/dma-mapping.h>
	#include <linux/slab.h>
	#include <linux/version.h>
	#include <linux/platform_device.h>
	#if defined(CONFIG_DMA_SHARED_BUFFER)
	#include <linux/dma-buf.h>
	#endif
	#include <linux/idr.h>

	#include "mali_osk.h"
	#include "mali_osk_mali.h"
	#include "mali_kernel_linux.h"
	#include "mali_scheduler.h"
	#include "mali_memory.h"
	#include "mali_memory_os_alloc.h"
	#if defined(CONFIG_DMA_SHARED_BUFFER)
	#include "mali_memory_dma_buf.h"
	#endif
	#if defined(CONFIG_MALI400_UMP)
	#include "mali_memory_ump.h"
	#endif
	#include "mali_memory_manager.h"
	#include "mali_memory_virtual.h"
	#include "mali_memory_util.h"
	#include "mali_memory_external.h"
	#include "mali_memory_block_alloc.h"

	#define MALI_S32_MAX 0x7fffffff

	/*
	* New memory system interface
	*/

	/inti idr for backend memory /
	struct idr mali_backend_idr;
	struct mutex mali_idr_mutex;

	/* init allocation manager */
	int mali_memory_manager_init(struct mali_allocation_manager *mgr)
	{
	/* init Locks */
	rwlock_init(&mgr->vm_lock);
	mutex_init(&mgr->list_mutex);

	/* init link */
	INIT_LIST_HEAD(&mgr->head);

	/* init RB tree */
	mgr->allocation_mgr_rb = RB_ROOT;
	mgr->mali_allocation_nr = 0;
	return 0;
	}

	/* deinit allocation manager
	* Do some check for debug
	*/
	void mali_memory_manager_uninit(struct mali_allocation_manager *mgr)
	{
	/* check RB tree is empty */
	MALI_DEBUG_ASSERT(((void )(mgr->allocation_mgr_rb.rb_node) == (void )rb_last(&mgr->allocation_mgr_rb)));
	/* check allocation List */
	MALI_DEBUG_ASSERT(list_empty(&mgr->head));
	}

	/* Prepare memory descriptor */
	static mali_mem_allocation mali_mem_allocation_struct_create(struct mali_session_data session)
	{
	mali_mem_allocation *mali_allocation;

	/* Allocate memory */
	mali_allocation = (mali_mem_allocation *)kzalloc(sizeof(mali_mem_allocation), GFP_KERNEL);
	if (NULL == mali_allocation) {
	MALI_DEBUG_PRINT(1, ("mali_mem_allocation_struct_create: descriptor was NULL\n"));
	return NULL;
	}

	MALI_DEBUG_CODE(mali_allocation->magic = MALI_MEM_ALLOCATION_VALID_MAGIC);

	/* do init */
	mali_allocation->flags = 0;
	mali_allocation->session = session;

	INIT_LIST_HEAD(&mali_allocation->list);
	kref_init(&mali_allocation->ref);

	/**
	*add to session list
	*/
	mutex_lock(&session->allocation_mgr.list_mutex);
	list_add_tail(&mali_allocation->list, &session->allocation_mgr.head);
	session->allocation_mgr.mali_allocation_nr++;
	mutex_unlock(&session->allocation_mgr.list_mutex);

	return mali_allocation;
	}


	void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc)
	{
	MALI_DEBUG_ASSERT_POINTER(alloc);
	MALI_DEBUG_ASSERT_POINTER(alloc->session);
	mutex_lock(&alloc->session->allocation_mgr.list_mutex);
	list_del(&alloc->list);
	alloc->session->allocation_mgr.mali_allocation_nr--;
	mutex_unlock(&alloc->session->allocation_mgr.list_mutex);

	kfree(alloc);
	}

	int mali_mem_backend_struct_create(mali_mem_backend **backend, u32 psize)
	{
	mali_mem_backend *mem_backend = NULL;
	s32 ret = -ENOSPC;
	s32 index = -1;
	backend = (mali_mem_backend )kzalloc(sizeof(mali_mem_backend), GFP_KERNEL);
	if (NULL == *backend) {
	MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: backend descriptor was NULL\n"));
	return -1;
	}
	mem_backend = *backend;
	mem_backend->size = psize;
	/* link backend with id */
	#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
	again:
	if (!idr_pre_get(&mali_backend_idr, GFP_KERNEL)) {
	kfree(mem_backend);
	return -ENOMEM;
	}
	mutex_lock(&mali_idr_mutex);
	ret = idr_get_new_above(&mali_backend_idr, mem_backend, 1, &index);
	mutex_unlock(&mali_idr_mutex);

	if (-ENOSPC == ret) {
	kfree(mem_backend);
	return -ENOSPC;
	}
	if (-EAGAIN == ret)
	goto again;
	#else
	mutex_lock(&mali_idr_mutex);
	ret = idr_alloc(&mali_backend_idr, mem_backend, 1, MALI_S32_MAX, GFP_KERNEL);
	mutex_unlock(&mali_idr_mutex);
	index = ret;
	if (ret < 0) {
	MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: Can't allocate idr for backend! \n"));
	kfree(mem_backend);
	return -ENOSPC;
	}
	#endif
	return index;
	}

	static void mali_mem_backend_struct_destory(mali_mem_backend **backend, s32 backend_handle)
	{
	mali_mem_backend mem_backend = backend;

	mutex_lock(&mali_idr_mutex);
	idr_remove(&mali_backend_idr, backend_handle);
	mutex_unlock(&mali_idr_mutex);
	kfree(mem_backend);
	*backend = NULL;
	}


	/* Set GPU MMU properties */
	static void _mali_memory_gpu_map_property_set(u32 *properties, u32 flags)
	{

	if (_MALI_MEMORY_GPU_READ_ALLOCATE & flags) {
	*properties = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE;
	} else {
	*properties = MALI_MMU_FLAGS_DEFAULT;
	}
	}


	/**
	* function@_mali_ukk_mem_allocate - allocate mali memory
	*/
	_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args)
	{
	struct mali_session_data session = (struct mali_session_data )(uintptr_t)args->ctx;
	mali_mem_backend *mem_backend = NULL;
	_mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
	int retval = 0;
	mali_mem_allocation *mali_allocation = NULL;
	struct mali_vma_node *mali_vma_node = NULL;

	MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_allocate, vaddr=0x%x, size =0x%x! \n", args->gpu_vaddr, args->psize));

	/* Check if the address is allocated
	* Can we trust User mode?
	*/
	mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->gpu_vaddr, 0);
	if (unlikely(mali_vma_node)) {
	/* Not support yet */
	MALI_DEBUG_ASSERT(0);
	return _MALI_OSK_ERR_FAULT;
	}

	/**
	*create mali memory allocation
	*/
	mali_allocation = mali_mem_allocation_struct_create(session);

	if (mali_allocation == NULL) {
	MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_allocate: Failed to create allocation struct! \n"));
	return _MALI_OSK_ERR_NOMEM;
	}
	mali_allocation->psize = args->psize;
	mali_allocation->vsize = args->vsize;

	/* check if have dedicated memory */
	if (MALI_TRUE == mali_memory_have_dedicated_memory()) {
	mali_allocation->type = MALI_MEM_BLOCK;
	} else {
	mali_allocation->type = MALI_MEM_OS;
	}

	/**
	*add allocation node to RB tree for index
	*/
	mali_allocation->mali_vma_node.vm_node.start = args->gpu_vaddr;
	mali_allocation->mali_vma_node.vm_node.size = args->vsize;

	mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node);

	/* check if need to allocate backend */
	if (mali_allocation->psize == 0)
	return _MALI_OSK_ERR_OK;

	/**
	*allocate physical backend & pages
	*/
	if (likely(mali_allocation->psize > 0)) {
	mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, args->psize);
	if (mali_allocation->backend_handle < 0) {
	ret = _MALI_OSK_ERR_NOMEM;
	MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n"));
	goto failed_alloc_backend;
	}

	mem_backend->mali_allocation = mali_allocation;
	mem_backend->type = mali_allocation->type;

	if (mem_backend->type == MALI_MEM_OS) {
	retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size);
	} else if (mem_backend->type == MALI_MEM_BLOCK) {
	/* try to allocated from BLOCK memory first, then try OS memory if failed.*/
	if (mali_mem_block_alloc(&mem_backend->block_mem, mem_backend->size)) {
	retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size);
	mem_backend->type = MALI_MEM_OS;
	mali_allocation->type = MALI_MEM_OS;
	}
	} else {
	/* ONLY support mem_os type */
	MALI_DEBUG_ASSERT(0);
	}

	if (retval) {
	ret = _MALI_OSK_ERR_NOMEM;
	MALI_DEBUG_PRINT(1, (" can't allocate enough pages! \n"));
	goto failed_alloc_pages;
	}
	}

	/**
	*map to GPU side
	*/
	mali_allocation->mali_mapping.addr = args->gpu_vaddr;

	/* set gpu mmu propery */
	_mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags);

	if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) {
	_mali_osk_mutex_wait(session->memory_lock);
	/* Map on Mali */
	ret = mali_mem_mali_map_prepare(mali_allocation);
	if (0 != ret) {
	MALI_DEBUG_PRINT(1, (" prepare map fail! \n"));
	goto failed_gpu_map;
	}
	/* only support os memory type now */
	if (mem_backend->type == MALI_MEM_OS) {
	mali_mem_os_mali_map(mem_backend, args->gpu_vaddr,
	mali_allocation->mali_mapping.properties);
	} else if (mem_backend->type == MALI_MEM_BLOCK) {
	mali_mem_block_mali_map(&mem_backend->block_mem, session, args->gpu_vaddr,
	mali_allocation->mali_mapping.properties);
	} else {
	/* Not support yet */
	MALI_DEBUG_ASSERT(0);
	}
	session->mali_mem_array[mem_backend->type] += mem_backend->size;
	if (session->mali_mem_array[MALI_MEM_OS] + session->mali_mem_array[MALI_MEM_BLOCK] > session->max_mali_mem_allocated) {
	session->max_mali_mem_allocated = session->mali_mem_array[MALI_MEM_OS] + session->mali_mem_array[MALI_MEM_BLOCK];
	}
	_mali_osk_mutex_signal(session->memory_lock);
	}

	return _MALI_OSK_ERR_OK;

	failed_gpu_map:
	_mali_osk_mutex_signal(session->memory_lock);
	if (mem_backend->type == MALI_MEM_OS) {
	mali_mem_os_free(&mem_backend->os_mem);
	} else {
	mali_mem_block_free(&mem_backend->block_mem);
	}
	failed_alloc_pages:
	mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle);
	failed_alloc_backend:

	mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node);
	mali_mem_allocation_struct_destory(mali_allocation);

	return ret;
	}


	_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args)
	{
	struct mali_session_data session = (struct mali_session_data )(uintptr_t)args->ctx;
	u32 vaddr = args->gpu_vaddr;
	mali_mem_allocation *mali_alloc = NULL;
	struct mali_vma_node *mali_vma_node = NULL;

	/* find mali allocation structure by vaddress*/
	mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, vaddr, 0);

	MALI_DEBUG_ASSERT(NULL != mali_vma_node);
	mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);

	if (mali_alloc)
	/* check ref_count */
	mali_allocation_unref(&mali_alloc);

	return _MALI_OSK_ERR_OK;
	}


	/**
	* Function _mali_ukk_mem_bind -- bind a external memory to a new GPU address
	* It will allocate a new mem allocation and bind external memory to it.
	* Supported backend type are:
	* _MALI_MEMORY_BIND_BACKEND_UMP
	* _MALI_MEMORY_BIND_BACKEND_DMA_BUF
	* _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY
	* CPU access is not supported yet
	*/
	_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args)
	{
	struct mali_session_data session = (struct mali_session_data )(uintptr_t)args->ctx;
	mali_mem_backend *mem_backend = NULL;
	_mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
	mali_mem_allocation *mali_allocation = NULL;
	MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_bind, vaddr=0x%x, size =0x%x! \n", args->vaddr, args->size));

	/**
	* allocate mali allocation.
	*/
	mali_allocation = mali_mem_allocation_struct_create(session);

	if (mali_allocation == NULL) {
	return _MALI_OSK_ERR_NOMEM;
	}
	mali_allocation->psize = args->size;
	mali_allocation->vsize = args->size;
	mali_allocation->mali_mapping.addr = args->vaddr;

	/* add allocation node to RB tree for index */
	mali_allocation->mali_vma_node.vm_node.start = args->vaddr;
	mali_allocation->mali_vma_node.vm_node.size = args->size;
	mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node);

	/* allocate backend*/
	if (mali_allocation->psize > 0) {
	mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize);
	if (mali_allocation->backend_handle < 0) {
	goto Failed_alloc_backend;
	}

	} else {
	goto Failed_alloc_backend;
	}

	mem_backend->size = mali_allocation->psize;
	mem_backend->mali_allocation = mali_allocation;

	switch (args->flags & _MALI_MEMORY_BIND_BACKEND_MASK) {
	case _MALI_MEMORY_BIND_BACKEND_UMP:
	#if defined(CONFIG_MALI400_UMP)
	mali_allocation->type = MALI_MEM_UMP;
	mem_backend->type = MALI_MEM_UMP;
	ret = mali_memory_bind_ump_buf(mali_allocation, mem_backend,
	args->mem_union.bind_ump.secure_id, args->mem_union.bind_ump.flags);
	if (_MALI_OSK_ERR_OK != ret) {
	MALI_DEBUG_PRINT(1, ("Bind ump buf failed\n"));
	goto Failed_bind_backend;
	}
	#else
	MALI_DEBUG_PRINT(1, ("UMP not supported\n"));
	goto Failed_bind_backend;
	#endif
	break;
	case _MALI_MEMORY_BIND_BACKEND_DMA_BUF:
	#if defined(CONFIG_DMA_SHARED_BUFFER)
	mali_allocation->type = MALI_MEM_DMA_BUF;
	mem_backend->type = MALI_MEM_DMA_BUF;
	ret = mali_memory_bind_dma_buf(mali_allocation, mem_backend,
	args->mem_union.bind_dma_buf.mem_fd, args->mem_union.bind_dma_buf.flags);
	if (_MALI_OSK_ERR_OK != ret) {
	MALI_DEBUG_PRINT(1, ("Bind dma buf failed\n"));
	goto Failed_bind_backend;
	}
	#else
	MALI_DEBUG_PRINT(1, ("DMA not supported\n"));
	goto Failed_bind_backend;
	#endif
	break;
	case _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY:
	/* not allowed */
	MALI_DEBUG_ASSERT(0);
	break;

	case _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY:
	mali_allocation->type = MALI_MEM_EXTERNAL;
	mem_backend->type = MALI_MEM_EXTERNAL;
	ret = mali_memory_bind_ext_mem(mali_allocation, mem_backend, args->mem_union.bind_ext_memory.phys_addr,
	args->mem_union.bind_ext_memory.flags);
	if (_MALI_OSK_ERR_OK != ret) {
	MALI_DEBUG_PRINT(1, ("Bind external buf failed\n"));
	goto Failed_bind_backend;
	}
	break;

	case _MALI_MEMORY_BIND_BACKEND_EXT_COW:
	/* not allowed */
	MALI_DEBUG_ASSERT(0);
	break;

	default:
	MALI_DEBUG_ASSERT(0);
	break;
	}
	return _MALI_OSK_ERR_OK;


	Failed_bind_backend:
	mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle);

	Failed_alloc_backend:
	mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node);
	mali_mem_allocation_struct_destory(mali_allocation);

	MALI_DEBUG_PRINT(1, (" _mali_ukk_mem_bind, return ERROR! \n"));
	return ret;
	}


	/*
	* Function _mali_ukk_mem_unbind -- unbind a external memory to a new GPU address
	* This function unbind the backend memory and free the allocation
	* no ref_count for this type of memory
	*/
	_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args)
	{
	/**/
	struct mali_session_data session = (struct mali_session_data )(uintptr_t)args->ctx;
	mali_mem_allocation *mali_allocation = NULL;
	struct mali_vma_node *mali_vma_node = NULL;
	u32 mali_addr = args->vaddr;
	MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_unbind, vaddr=0x%x! \n", args->vaddr));

	/* find the allocation by vaddr */
	mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
	if (likely(mali_vma_node)) {
	MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start);
	mali_allocation = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
	} else {
	MALI_DEBUG_ASSERT(NULL != mali_vma_node);
	/* Not support yet */
	MALI_DEBUG_ASSERT(0);
	return _MALI_OSK_ERR_INVALID_ARGS;
	}

	if (NULL != mali_allocation)
	/* check ref_count */
	mali_allocation_unref(&mali_allocation);
	return _MALI_OSK_ERR_OK;
	}



	/*
	* Function _mali_ukk_mem_cow -- COW for an allocation
	* This function allocate new pages for a range (range, range+size) of allocation
	* And Map it(keep use the not in range pages from target allocation ) to an GPU vaddr
	*/
	_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args)
	{
	_mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;

	/* create new alloction if needed */

	/* Get the target allocation and it's backend*/

	/* allocate new pages from os mem for modified range */


	/* fill the COW backend, all pages for this allocation
	* including the new page for modified range and pages not modified in old allocation.
	* Do Add ref to pages from target allocation
	*/


	/* map it to GPU side */
	return ret;
	}

	/**
	* attach a backend to an exist mali allocation
	*/


	/**
	* deattach a backend from an exist mali allocation
	*/