mali/linux/mali_memory_defer_bind.c - manifest_repos/mali-driver - Git at Google

 /*
  * Copyright (C) 2013-2016 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
  *
  * A copy of the licence is included with the program, and can also be obtained from Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 #include <linux/mm.h>
 #include <linux/list.h>
 #include <linux/mm_types.h>
 #include <linux/fs.h>
 #include <linux/dma-mapping.h>
 #include <linux/highmem.h>
 #include <asm/cacheflush.h>
 #include <linux/sched.h>
 #ifdef CONFIG_ARM
 #include <asm/outercache.h>
 #endif
 #include <asm/dma-mapping.h>

 #include "mali_memory.h"
 #include "mali_kernel_common.h"
 #include "mali_uk_types.h"
 #include "mali_osk.h"
 #include "mali_kernel_linux.h"
 #include "mali_memory_defer_bind.h"
 #include "mali_executor.h"
 #include "mali_osk.h"
 #include "mali_scheduler.h"
 #include "mali_gp_job.h"

 mali_defer_bind_manager *mali_dmem_man = NULL;

 static u32 mali_dmem_get_gp_varying_size(struct mali_gp_job *gp_job)
 {
 	return gp_job->required_varying_memsize / _MALI_OSK_MALI_PAGE_SIZE;
 }

 _mali_osk_errcode_t mali_mem_defer_bind_manager_init(void)
 {
 	mali_dmem_man = _mali_osk_calloc(1, sizeof(struct mali_defer_bind_manager));
 	if (!mali_dmem_man)
 		return _MALI_OSK_ERR_NOMEM;

 	atomic_set(&mali_dmem_man->num_used_pages, 0);
 	atomic_set(&mali_dmem_man->num_dmem, 0);

 	return _MALI_OSK_ERR_OK;
 }


 void mali_mem_defer_bind_manager_destory(void)
 {
 	if (mali_dmem_man) {
 		MALI_DEBUG_ASSERT(0 == atomic_read(&mali_dmem_man->num_dmem));
 		kfree(mali_dmem_man);
 	}
 	mali_dmem_man = NULL;
 }


 /*allocate pages from OS memory*/
 _mali_osk_errcode_t mali_mem_defer_alloc_mem(u32 require, struct mali_session_data *session, mali_defer_mem_block *dblock)
 {
 	int retval = 0;
 	u32 num_pages = require;
 	mali_mem_os_mem os_mem;

 	retval = mali_mem_os_alloc_pages(&os_mem, num_pages * _MALI_OSK_MALI_PAGE_SIZE);

 	/* add to free pages list */
 	if (0 == retval) {
 		MALI_DEBUG_PRINT(4, ("mali_mem_defer_alloc_mem ,,*** pages allocate = 0x%x \n", num_pages));
 		list_splice(&os_mem.pages, &dblock->free_pages);
 		atomic_add(os_mem.count, &dblock->num_free_pages);
 		atomic_add(os_mem.count, &session->mali_mem_allocated_pages);
 		if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) {
 			session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE;
 		}
 		return _MALI_OSK_ERR_OK;
 	} else
 		return _MALI_OSK_ERR_FAULT;
 }

 _mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock)
 {
 	u32 require_page;

 	if (!next_gp_job)
 		return _MALI_OSK_ERR_FAULT;

 	require_page = mali_dmem_get_gp_varying_size(next_gp_job);

 	MALI_DEBUG_PRINT(4, ("mali_mem_defer_prepare_mem_work, require alloc page 0x%x\n",
 			     require_page));
 	/* allocate more pages from OS */
 	if (_MALI_OSK_ERR_OK != mali_mem_defer_alloc_mem(require_page, next_gp_job->session, dblock)) {
 		MALI_DEBUG_PRINT(1, ("ERROR##mali_mem_defer_prepare_mem_work, allocate page failed!!"));
 		return _MALI_OSK_ERR_NOMEM;
 	}

 	next_gp_job->bind_flag = MALI_DEFER_BIND_MEMORY_PREPARED;

 	return _MALI_OSK_ERR_OK;
 }


 /* do preparetion for allocation before defer bind */
 _mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list, u32 *required_varying_memsize)
 {
 	mali_mem_backend *mem_bkend = NULL;
 	struct mali_backend_bind_list *bk_list = _mali_osk_calloc(1, sizeof(struct mali_backend_bind_list));
 	if (NULL == bk_list)
 		return _MALI_OSK_ERR_FAULT;

 	INIT_LIST_HEAD(&bk_list->node);
 	/* Get backend memory */
 	mutex_lock(&mali_idr_mutex);
 	if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) {
 		MALI_DEBUG_PRINT(1, ("Can't find memory backend in defer bind!\n"));
 		mutex_unlock(&mali_idr_mutex);
 		_mali_osk_free(bk_list);
 		return _MALI_OSK_ERR_FAULT;
 	}
 	mutex_unlock(&mali_idr_mutex);

 	/* If the mem backend has already been bound, no need to bind again.*/
 	if (mem_bkend->os_mem.count > 0) {
 		_mali_osk_free(bk_list);
 		return _MALI_OSK_ERR_OK;
 	}

 	MALI_DEBUG_PRINT(4, ("bind_allocation_prepare:: allocation =%x vaddr=0x%x!\n", alloc, alloc->mali_vma_node.vm_node.start));

 	INIT_LIST_HEAD(&mem_bkend->os_mem.pages);

 	bk_list->bkend = mem_bkend;
 	bk_list->vaddr = alloc->mali_vma_node.vm_node.start;
 	bk_list->session = alloc->session;
 	bk_list->page_num = mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE;
 	*required_varying_memsize +=  mem_bkend->size;
 	MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS);

 	/* add to job to do list */
 	list_add(&bk_list->node, list);

 	return _MALI_OSK_ERR_OK;
 }


 /* bind phyiscal memory to allocation
 This function will be called in IRQ handler*/
 static _mali_osk_errcode_t mali_mem_defer_bind_allocation(struct mali_backend_bind_list *bk_node,
 		struct list_head *pages)
 {
 	struct mali_session_data *session = bk_node->session;
 	mali_mem_backend *mem_bkend = bk_node->bkend;
 	MALI_DEBUG_PRINT(4, ("mali_mem_defer_bind_allocation, bind bkend = %x page num=0x%x vaddr=%x session=%x\n", mem_bkend, bk_node->page_num, bk_node->vaddr, session));

 	MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS);
 	list_splice(pages, &mem_bkend->os_mem.pages);
 	mem_bkend->os_mem.count = bk_node->page_num;

 	if (mem_bkend->type == MALI_MEM_OS) {
 		mali_mem_os_mali_map(&mem_bkend->os_mem, session, bk_node->vaddr, 0,
 				     mem_bkend->os_mem.count, MALI_MMU_FLAGS_DEFAULT);
 	}
 	smp_wmb();
 	bk_node->flag = MALI_DEFER_BIND_MEMORY_BINDED;
 	mem_bkend->flags &= ~MALI_MEM_BACKEND_FLAG_NOT_BINDED;
 	mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_BINDED;
 	return _MALI_OSK_ERR_OK;
 }


 static struct list_head *mali_mem_defer_get_free_page_list(u32 count, struct list_head *pages, mali_defer_mem_block *dblock)
 {
 	int i = 0;
 	struct mali_page_node *m_page, *m_tmp;

 	if (atomic_read(&dblock->num_free_pages) < count) {
 		return NULL;
 	} else {
 		list_for_each_entry_safe(m_page, m_tmp, &dblock->free_pages, list) {
 			if (i < count) {
 				list_move_tail(&m_page->list, pages);
 			} else {
 				break;
 			}
 			i++;
 		}
 		MALI_DEBUG_ASSERT(i == count);
 		atomic_sub(count, &dblock->num_free_pages);
 		return pages;
 	}
 }


 /* called in job start IOCTL to bind physical memory for each allocations
 @ bk_list backend list to do defer bind
 @ pages page list to do this bind
 @ count number of pages
 */
 _mali_osk_errcode_t mali_mem_defer_bind(struct mali_gp_job *gp,
 					struct mali_defer_mem_block *dmem_block)
 {
 	struct mali_defer_mem *dmem = NULL;
 	struct mali_backend_bind_list *bkn, *bkn_tmp;
 	LIST_HEAD(pages);

 	if (gp->required_varying_memsize != (atomic_read(&dmem_block->num_free_pages) * _MALI_OSK_MALI_PAGE_SIZE)) {
 		MALI_DEBUG_PRINT_ERROR(("#BIND:  The memsize of varying buffer not match to the pagesize of the dmem_block!!## \n"));
 		return _MALI_OSK_ERR_FAULT;
 	}

 	MALI_DEBUG_PRINT(4, ("#BIND: GP job=%x## \n", gp));
 	dmem = (mali_defer_mem *)_mali_osk_calloc(1, sizeof(struct mali_defer_mem));
 	if (dmem) {
 		INIT_LIST_HEAD(&dmem->node);
 		gp->dmem = dmem;
 	} else {
 		return _MALI_OSK_ERR_NOMEM;
 	}

 	atomic_add(1, &mali_dmem_man->num_dmem);
 	/* for each bk_list backend, do bind */
 	list_for_each_entry_safe(bkn, bkn_tmp , &gp->vary_todo, node) {
 		INIT_LIST_HEAD(&pages);
 		if (likely(mali_mem_defer_get_free_page_list(bkn->page_num, &pages, dmem_block))) {
 			list_del(&bkn->node);
 			mali_mem_defer_bind_allocation(bkn, &pages);
 			_mali_osk_free(bkn);
 		} else {
 			/* not enough memory will not happen */
 			MALI_DEBUG_PRINT_ERROR(("#BIND: NOT enough memory when binded !!## \n"));
 			_mali_osk_free(gp->dmem);
 			return _MALI_OSK_ERR_NOMEM;
 		}
 	}

 	if (!list_empty(&gp->vary_todo)) {
 		MALI_DEBUG_PRINT_ERROR(("#BIND:  The deferbind backend list isn't empty !!## \n"));
 		_mali_osk_free(gp->dmem);
 		return _MALI_OSK_ERR_FAULT;
 	}

 	dmem->flag = MALI_DEFER_BIND_MEMORY_BINDED;

 	return _MALI_OSK_ERR_OK;
 }

 void mali_mem_defer_dmem_free(struct mali_gp_job *gp)
 {
 	if (gp->dmem) {
 		atomic_dec(&mali_dmem_man->num_dmem);
 		_mali_osk_free(gp->dmem);
 	}
 }
	/*
	* Copyright (C) 2013-2016 ARM Limited. All rights reserved.
	*
	* This program is free software and is provided to you under the terms of the GNU General Public License version 2
	* as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
	*
	* A copy of the licence is included with the program, and can also be obtained from Free Software
	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
	*/
	#include <linux/mm.h>
	#include <linux/list.h>
	#include <linux/mm_types.h>
	#include <linux/fs.h>
	#include <linux/dma-mapping.h>
	#include <linux/highmem.h>
	#include <asm/cacheflush.h>
	#include <linux/sched.h>
	#ifdef CONFIG_ARM
	#include <asm/outercache.h>
	#endif
	#include <asm/dma-mapping.h>

	#include "mali_memory.h"
	#include "mali_kernel_common.h"
	#include "mali_uk_types.h"
	#include "mali_osk.h"
	#include "mali_kernel_linux.h"
	#include "mali_memory_defer_bind.h"
	#include "mali_executor.h"
	#include "mali_osk.h"
	#include "mali_scheduler.h"
	#include "mali_gp_job.h"

	mali_defer_bind_manager *mali_dmem_man = NULL;

	static u32 mali_dmem_get_gp_varying_size(struct mali_gp_job *gp_job)
	{
	return gp_job->required_varying_memsize / _MALI_OSK_MALI_PAGE_SIZE;
	}

	_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void)
	{
	mali_dmem_man = _mali_osk_calloc(1, sizeof(struct mali_defer_bind_manager));
	if (!mali_dmem_man)
	return _MALI_OSK_ERR_NOMEM;

	atomic_set(&mali_dmem_man->num_used_pages, 0);
	atomic_set(&mali_dmem_man->num_dmem, 0);

	return _MALI_OSK_ERR_OK;
	}


	void mali_mem_defer_bind_manager_destory(void)
	{
	if (mali_dmem_man) {
	MALI_DEBUG_ASSERT(0 == atomic_read(&mali_dmem_man->num_dmem));
	kfree(mali_dmem_man);
	}
	mali_dmem_man = NULL;
	}


	/allocate pages from OS memory/
	_mali_osk_errcode_t mali_mem_defer_alloc_mem(u32 require, struct mali_session_data session, mali_defer_mem_block dblock)
	{
	int retval = 0;
	u32 num_pages = require;
	mali_mem_os_mem os_mem;

	retval = mali_mem_os_alloc_pages(&os_mem, num_pages * _MALI_OSK_MALI_PAGE_SIZE);

	/* add to free pages list */
	if (0 == retval) {
	MALI_DEBUG_PRINT(4, ("mali_mem_defer_alloc_mem ,,*** pages allocate = 0x%x \n", num_pages));
	list_splice(&os_mem.pages, &dblock->free_pages);
	atomic_add(os_mem.count, &dblock->num_free_pages);
	atomic_add(os_mem.count, &session->mali_mem_allocated_pages);
	if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) {
	session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE;
	}
	return _MALI_OSK_ERR_OK;
	} else
	return _MALI_OSK_ERR_FAULT;
	}

	_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job next_gp_job, mali_defer_mem_block dblock)
	{
	u32 require_page;

	if (!next_gp_job)
	return _MALI_OSK_ERR_FAULT;

	require_page = mali_dmem_get_gp_varying_size(next_gp_job);

	MALI_DEBUG_PRINT(4, ("mali_mem_defer_prepare_mem_work, require alloc page 0x%x\n",
	require_page));
	/* allocate more pages from OS */
	if (_MALI_OSK_ERR_OK != mali_mem_defer_alloc_mem(require_page, next_gp_job->session, dblock)) {
	MALI_DEBUG_PRINT(1, ("ERROR##mali_mem_defer_prepare_mem_work, allocate page failed!!"));
	return _MALI_OSK_ERR_NOMEM;
	}

	next_gp_job->bind_flag = MALI_DEFER_BIND_MEMORY_PREPARED;

	return _MALI_OSK_ERR_OK;
	}


	/* do preparetion for allocation before defer bind */
	_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation alloc, struct list_head list, u32 *required_varying_memsize)
	{
	mali_mem_backend *mem_bkend = NULL;
	struct mali_backend_bind_list *bk_list = _mali_osk_calloc(1, sizeof(struct mali_backend_bind_list));
	if (NULL == bk_list)
	return _MALI_OSK_ERR_FAULT;

	INIT_LIST_HEAD(&bk_list->node);
	/* Get backend memory */
	mutex_lock(&mali_idr_mutex);
	if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) {
	MALI_DEBUG_PRINT(1, ("Can't find memory backend in defer bind!\n"));
	mutex_unlock(&mali_idr_mutex);
	_mali_osk_free(bk_list);
	return _MALI_OSK_ERR_FAULT;
	}
	mutex_unlock(&mali_idr_mutex);

	/* If the mem backend has already been bound, no need to bind again.*/
	if (mem_bkend->os_mem.count > 0) {
	_mali_osk_free(bk_list);
	return _MALI_OSK_ERR_OK;
	}

	MALI_DEBUG_PRINT(4, ("bind_allocation_prepare:: allocation =%x vaddr=0x%x!\n", alloc, alloc->mali_vma_node.vm_node.start));

	INIT_LIST_HEAD(&mem_bkend->os_mem.pages);

	bk_list->bkend = mem_bkend;
	bk_list->vaddr = alloc->mali_vma_node.vm_node.start;
	bk_list->session = alloc->session;
	bk_list->page_num = mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE;
	*required_varying_memsize += mem_bkend->size;
	MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS);

	/* add to job to do list */
	list_add(&bk_list->node, list);

	return _MALI_OSK_ERR_OK;
	}



	/* bind phyiscal memory to allocation
	This function will be called in IRQ handler*/
	static _mali_osk_errcode_t mali_mem_defer_bind_allocation(struct mali_backend_bind_list *bk_node,
	struct list_head *pages)
	{
	struct mali_session_data *session = bk_node->session;
	mali_mem_backend *mem_bkend = bk_node->bkend;
	MALI_DEBUG_PRINT(4, ("mali_mem_defer_bind_allocation, bind bkend = %x page num=0x%x vaddr=%x session=%x\n", mem_bkend, bk_node->page_num, bk_node->vaddr, session));

	MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS);
	list_splice(pages, &mem_bkend->os_mem.pages);
	mem_bkend->os_mem.count = bk_node->page_num;

	if (mem_bkend->type == MALI_MEM_OS) {
	mali_mem_os_mali_map(&mem_bkend->os_mem, session, bk_node->vaddr, 0,
	mem_bkend->os_mem.count, MALI_MMU_FLAGS_DEFAULT);
	}
	smp_wmb();
	bk_node->flag = MALI_DEFER_BIND_MEMORY_BINDED;
	mem_bkend->flags &= ~MALI_MEM_BACKEND_FLAG_NOT_BINDED;
	mem_bkend->flags \|= MALI_MEM_BACKEND_FLAG_BINDED;
	return _MALI_OSK_ERR_OK;
	}


	static struct list_head mali_mem_defer_get_free_page_list(u32 count, struct list_head pages, mali_defer_mem_block *dblock)
	{
	int i = 0;
	struct mali_page_node m_page, m_tmp;

	if (atomic_read(&dblock->num_free_pages) < count) {
	return NULL;
	} else {
	list_for_each_entry_safe(m_page, m_tmp, &dblock->free_pages, list) {
	if (i < count) {
	list_move_tail(&m_page->list, pages);
	} else {
	break;
	}
	i++;
	}
	MALI_DEBUG_ASSERT(i == count);
	atomic_sub(count, &dblock->num_free_pages);
	return pages;
	}
	}


	/* called in job start IOCTL to bind physical memory for each allocations
	@ bk_list backend list to do defer bind
	@ pages page list to do this bind
	@ count number of pages
	*/
	_mali_osk_errcode_t mali_mem_defer_bind(struct mali_gp_job *gp,
	struct mali_defer_mem_block *dmem_block)
	{
	struct mali_defer_mem *dmem = NULL;
	struct mali_backend_bind_list bkn, bkn_tmp;
	LIST_HEAD(pages);

	if (gp->required_varying_memsize != (atomic_read(&dmem_block->num_free_pages) * _MALI_OSK_MALI_PAGE_SIZE)) {
	MALI_DEBUG_PRINT_ERROR(("#BIND: The memsize of varying buffer not match to the pagesize of the dmem_block!!## \n"));
	return _MALI_OSK_ERR_FAULT;
	}

	MALI_DEBUG_PRINT(4, ("#BIND: GP job=%x## \n", gp));
	dmem = (mali_defer_mem *)_mali_osk_calloc(1, sizeof(struct mali_defer_mem));
	if (dmem) {
	INIT_LIST_HEAD(&dmem->node);
	gp->dmem = dmem;
	} else {
	return _MALI_OSK_ERR_NOMEM;
	}

	atomic_add(1, &mali_dmem_man->num_dmem);
	/* for each bk_list backend, do bind */
	list_for_each_entry_safe(bkn, bkn_tmp , &gp->vary_todo, node) {
	INIT_LIST_HEAD(&pages);
	if (likely(mali_mem_defer_get_free_page_list(bkn->page_num, &pages, dmem_block))) {
	list_del(&bkn->node);
	mali_mem_defer_bind_allocation(bkn, &pages);
	_mali_osk_free(bkn);
	} else {
	/* not enough memory will not happen */
	MALI_DEBUG_PRINT_ERROR(("#BIND: NOT enough memory when binded !!## \n"));
	_mali_osk_free(gp->dmem);
	return _MALI_OSK_ERR_NOMEM;
	}
	}

	if (!list_empty(&gp->vary_todo)) {
	MALI_DEBUG_PRINT_ERROR(("#BIND: The deferbind backend list isn't empty !!## \n"));
	_mali_osk_free(gp->dmem);
	return _MALI_OSK_ERR_FAULT;
	}

	dmem->flag = MALI_DEFER_BIND_MEMORY_BINDED;

	return _MALI_OSK_ERR_OK;
	}

	void mali_mem_defer_dmem_free(struct mali_gp_job *gp)
	{
	if (gp->dmem) {
	atomic_dec(&mali_dmem_man->num_dmem);
	_mali_osk_free(gp->dmem);
	}
	}