mali/common/mali_mmu_page_directory.c - manifest_repos/mali-driver - Git at Google

 /*
  * Copyright (C) 2011-2016 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
  *
  * A copy of the licence is included with the program, and can also be obtained from Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */

 #include "mali_kernel_common.h"
 #include "mali_osk.h"
 #include "mali_ukk.h"
 #include "mali_uk_types.h"
 #include "mali_mmu_page_directory.h"
 #include "mali_memory.h"
 #include "mali_l2_cache.h"

 static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data);

 u32 mali_allocate_empty_page(mali_io_address *virt_addr)
 {
 	_mali_osk_errcode_t err;
 	mali_io_address mapping;
 	mali_dma_addr address;

 	if (_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping)) {
 		/* Allocation failed */
 		MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to get table page for empty pgdir\n"));
 		return 0;
 	}

 	MALI_DEBUG_ASSERT_POINTER(mapping);

 	err = fill_page(mapping, 0);
 	if (_MALI_OSK_ERR_OK != err) {
 		mali_mmu_release_table_page(address, mapping);
 		MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to zero page\n"));
 		return 0;
 	}

 	*virt_addr = mapping;
 	return address;
 }

 void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr)
 {
 	if (MALI_INVALID_PAGE != address) {
 		mali_mmu_release_table_page(address, virt_addr);
 	}
 }

 _mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory,
 		mali_io_address *page_directory_mapping,
 		mali_dma_addr *page_table, mali_io_address *page_table_mapping,
 		mali_dma_addr *data_page, mali_io_address *data_page_mapping)
 {
 	_mali_osk_errcode_t err;

 	err = mali_mmu_get_table_page(data_page, data_page_mapping);
 	if (_MALI_OSK_ERR_OK == err) {
 		err = mali_mmu_get_table_page(page_table, page_table_mapping);
 		if (_MALI_OSK_ERR_OK == err) {
 			err = mali_mmu_get_table_page(page_directory, page_directory_mapping);
 			if (_MALI_OSK_ERR_OK == err) {
 				fill_page(*data_page_mapping, 0);
 				fill_page(*page_table_mapping, *data_page | MALI_MMU_FLAGS_DEFAULT);
 				fill_page(*page_directory_mapping, *page_table | MALI_MMU_FLAGS_PRESENT);
 				MALI_SUCCESS;
 			}
 			mali_mmu_release_table_page(*page_table, *page_table_mapping);
 			*page_table = MALI_INVALID_PAGE;
 		}
 		mali_mmu_release_table_page(*data_page, *data_page_mapping);
 		*data_page = MALI_INVALID_PAGE;
 	}
 	return err;
 }

 void mali_destroy_fault_flush_pages(
 	mali_dma_addr *page_directory, mali_io_address *page_directory_mapping,
 	mali_dma_addr *page_table, mali_io_address *page_table_mapping,
 	mali_dma_addr *data_page, mali_io_address *data_page_mapping)
 {
 	if (MALI_INVALID_PAGE != *page_directory) {
 		mali_mmu_release_table_page(*page_directory, *page_directory_mapping);
 		*page_directory = MALI_INVALID_PAGE;
 		*page_directory_mapping = NULL;
 	}

 	if (MALI_INVALID_PAGE != *page_table) {
 		mali_mmu_release_table_page(*page_table, *page_table_mapping);
 		*page_table = MALI_INVALID_PAGE;
 		*page_table_mapping = NULL;
 	}

 	if (MALI_INVALID_PAGE != *data_page) {
 		mali_mmu_release_table_page(*data_page, *data_page_mapping);
 		*data_page = MALI_INVALID_PAGE;
 		*data_page_mapping = NULL;
 	}
 }

 static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data)
 {
 	int i;
 	MALI_DEBUG_ASSERT_POINTER(mapping);

 	for (i = 0; i < MALI_MMU_PAGE_SIZE / 4; i++) {
 		_mali_osk_mem_iowrite32_relaxed(mapping, i * sizeof(u32), data);
 	}
 	_mali_osk_mem_barrier();
 	MALI_SUCCESS;
 }

 _mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
 {
 	const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
 	const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
 	_mali_osk_errcode_t err;
 	mali_io_address pde_mapping;
 	mali_dma_addr pde_phys;
 	int i, page_count;
 	u32 start_address;
 	if (last_pde < first_pde)
 		return _MALI_OSK_ERR_INVALID_ARGS;

 	for (i = first_pde; i <= last_pde; i++) {
 		if (0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 						 i * sizeof(u32)) & MALI_MMU_FLAGS_PRESENT)) {
 			/* Page table not present */
 			MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
 			MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]);

 			err = mali_mmu_get_table_page(&pde_phys, &pde_mapping);
 			if (_MALI_OSK_ERR_OK != err) {
 				MALI_PRINT_ERROR(("Failed to allocate page table page.\n"));
 				return err;
 			}
 			pagedir->page_entries_mapped[i] = pde_mapping;

 			/* Update PDE, mark as present */
 			_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32),
 							pde_phys | MALI_MMU_FLAGS_PRESENT);

 			MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
 		}

 		if (first_pde == last_pde) {
 			pagedir->page_entries_usage_count[i] += size / MALI_MMU_PAGE_SIZE;
 		} else if (i == first_pde) {
 			start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE;
 			page_count = (start_address + MALI_MMU_VIRTUAL_PAGE_SIZE - mali_address) / MALI_MMU_PAGE_SIZE;
 			pagedir->page_entries_usage_count[i] += page_count;
 		} else if (i == last_pde) {
 			start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE;
 			page_count = (mali_address + size - start_address) / MALI_MMU_PAGE_SIZE;
 			pagedir->page_entries_usage_count[i] += page_count;
 		} else {
 			pagedir->page_entries_usage_count[i] = 1024;
 		}
 	}
 	_mali_osk_write_mem_barrier();

 	return _MALI_OSK_ERR_OK;
 }

 MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size)
 {
 	int i;
 	const int first_pte = MALI_MMU_PTE_ENTRY(mali_address);
 	const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1);

 	for (i = first_pte; i <= last_pte; i++) {
 		_mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0);
 	}
 }

 static u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index)
 {
 	return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 				       index * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK);
 }


 _mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
 {
 	const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
 	const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
 	u32 left = size;
 	int i;
 	mali_bool pd_changed = MALI_FALSE;
 	u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */
 	u32 num_pages_inv = 0;
 	mali_bool invalidate_all = MALI_FALSE; /* safety mechanism in case page_entries_usage_count is unreliable */

 	/* For all page directory entries in range. */
 	for (i = first_pde; i <= last_pde; i++) {
 		u32 size_in_pde, offset;

 		MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]);
 		MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]);

 		/* Offset into page table, 0 if mali_address is 4MiB aligned */
 		offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1));
 		if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset) {
 			size_in_pde = left;
 		} else {
 			size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset;
 		}

 		pagedir->page_entries_usage_count[i] -= size_in_pde / MALI_MMU_PAGE_SIZE;

 		/* If entire page table is unused, free it */
 		if (0 == pagedir->page_entries_usage_count[i]) {
 			u32 page_phys;
 			void *page_virt;
 			MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n"));
 			/* last reference removed, no need to zero out each PTE  */

 			page_phys = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i * sizeof(u32)));
 			page_virt = pagedir->page_entries_mapped[i];
 			pagedir->page_entries_mapped[i] = NULL;
 			_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);

 			mali_mmu_release_table_page(page_phys, page_virt);
 			pd_changed = MALI_TRUE;
 		} else {
 			MALI_DEBUG_ASSERT(num_pages_inv < 2);
 			if (num_pages_inv < 2) {
 				pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i);
 				num_pages_inv++;
 			} else {
 				invalidate_all = MALI_TRUE;
 			}

 			/* If part of the page table is still in use, zero the relevant PTEs */
 			mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde);
 		}

 		left -= size_in_pde;
 		mali_address += size_in_pde;
 	}
 	_mali_osk_write_mem_barrier();

 	/* L2 pages invalidation */
 	if (MALI_TRUE == pd_changed) {
 		MALI_DEBUG_ASSERT(num_pages_inv < 3);
 		if (num_pages_inv < 3) {
 			pages_to_invalidate[num_pages_inv] = pagedir->page_directory;
 			num_pages_inv++;
 		} else {
 			invalidate_all = MALI_TRUE;
 		}
 	}

 	if (invalidate_all) {
 		mali_l2_cache_invalidate_all();
 	} else {
 		mali_l2_cache_invalidate_all_pages(pages_to_invalidate, num_pages_inv);
 	}

 	MALI_SUCCESS;
 }

 struct mali_page_directory *mali_mmu_pagedir_alloc(void)
 {
 	struct mali_page_directory *pagedir;
 	_mali_osk_errcode_t err;
 	mali_dma_addr phys;

 	pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory));
 	if (NULL == pagedir) {
 		return NULL;
 	}

 	err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped);
 	if (_MALI_OSK_ERR_OK != err) {
 		_mali_osk_free(pagedir);
 		return NULL;
 	}

 	pagedir->page_directory = (u32)phys;

 	/* Zero page directory */
 	fill_page(pagedir->page_directory_mapped, 0);

 	return pagedir;
 }

 void mali_mmu_pagedir_free(struct mali_page_directory *pagedir)
 {
 	const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]);
 	int i;

 	/* Free referenced page tables and zero PDEs. */
 	for (i = 0; i < num_page_table_entries; i++) {
 		if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32(
 				pagedir->page_directory_mapped,
 				sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT)) {
 			mali_dma_addr phys = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 					     i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK;
 			_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);
 			mali_mmu_release_table_page(phys, pagedir->page_entries_mapped[i]);
 		}
 	}
 	_mali_osk_write_mem_barrier();

 	/* Free the page directory page. */
 	mali_mmu_release_table_page(pagedir->page_directory, pagedir->page_directory_mapped);

 	_mali_osk_free(pagedir);
 }


 void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address,
 			     mali_dma_addr phys_address, u32 size, u32 permission_bits)
 {
 	u32 end_address = mali_address + size;
 	u32 mali_phys = (u32)phys_address;

 	/* Map physical pages into MMU page tables */
 	for (; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, mali_phys += MALI_MMU_PAGE_SIZE) {
 		MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]);
 		_mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)],
 						MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32),
 						mali_phys | permission_bits);
 	}
 }

 void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr)
 {
 #if defined(DEBUG)
 	u32 pde_index, pte_index;
 	u32 pde, pte;

 	pde_index = MALI_MMU_PDE_ENTRY(fault_addr);
 	pte_index = MALI_MMU_PTE_ENTRY(fault_addr);


 	pde = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 				     pde_index * sizeof(u32));


 	if (pde & MALI_MMU_FLAGS_PRESENT) {
 		u32 pte_addr = MALI_MMU_ENTRY_ADDRESS(pde);

 		pte = _mali_osk_mem_ioread32(pagedir->page_entries_mapped[pde_index],
 					     pte_index * sizeof(u32));

 		MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table present: %08x\n"
 				     "\t\tPTE: %08x, page %08x is %s\n",
 				     fault_addr, pte_addr, pte,
 				     MALI_MMU_ENTRY_ADDRESS(pte),
 				     pte & MALI_MMU_FLAGS_DEFAULT ? "rw" : "not present"));
 	} else {
 		MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table not present: %08x\n",
 				     fault_addr, pde));
 	}
 #else
 	MALI_IGNORE(pagedir);
 	MALI_IGNORE(fault_addr);
 #endif
 }

 /* For instrumented */
 struct dump_info {
 	u32 buffer_left;
 	u32 register_writes_size;
 	u32 page_table_dump_size;
 	u32 *buffer;
 };

 static _mali_osk_errcode_t writereg(u32 where, u32 what, const char *comment, struct dump_info *info)
 {
 	if (NULL != info) {
 		info->register_writes_size += sizeof(u32) * 2; /* two 32-bit words */

 		if (NULL != info->buffer) {
 			/* check that we have enough space */
 			if (info->buffer_left < sizeof(u32) * 2) MALI_ERROR(_MALI_OSK_ERR_NOMEM);

 			*info->buffer = where;
 			info->buffer++;

 			*info->buffer = what;
 			info->buffer++;

 			info->buffer_left -= sizeof(u32) * 2;
 		}
 	}

 	MALI_SUCCESS;
 }

 static _mali_osk_errcode_t mali_mmu_dump_page(mali_io_address page, u32 phys_addr, struct dump_info *info)
 {
 	if (NULL != info) {
 		/* 4096 for the page and 4 bytes for the address */
 		const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4;
 		const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE;
 		const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4;

 		info->page_table_dump_size += dump_size_in_bytes;

 		if (NULL != info->buffer) {
 			if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM);

 			*info->buffer = phys_addr;
 			info->buffer++;

 			_mali_osk_memcpy(info->buffer, page, page_size_in_bytes);
 			info->buffer += page_size_in_elements;

 			info->buffer_left -= dump_size_in_bytes;
 		}
 	}

 	MALI_SUCCESS;
 }

 static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory *pagedir, struct dump_info *info)
 {
 	MALI_DEBUG_ASSERT_POINTER(pagedir);
 	MALI_DEBUG_ASSERT_POINTER(info);

 	if (NULL != pagedir->page_directory_mapped) {
 		int i;

 		MALI_CHECK_NO_ERROR(
 			mali_mmu_dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info)
 		);

 		for (i = 0; i < 1024; i++) {
 			if (NULL != pagedir->page_entries_mapped[i]) {
 				MALI_CHECK_NO_ERROR(
 					mali_mmu_dump_page(pagedir->page_entries_mapped[i],
 							   _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 									   i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info)
 				);
 			}
 		}
 	}

 	MALI_SUCCESS;
 }

 static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory *pagedir, struct dump_info *info)
 {
 	MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory,
 				     "set the page directory address", info));
 	MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info));
 	MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info));
 	MALI_SUCCESS;
 }

 _mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args)
 {
 	struct dump_info info = { 0, 0, 0, NULL };
 	struct mali_session_data *session_data;

 	session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
 	MALI_DEBUG_ASSERT_POINTER(session_data);
 	MALI_DEBUG_ASSERT_POINTER(args);

 	MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
 	MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
 	args->size = info.register_writes_size + info.page_table_dump_size;
 	MALI_SUCCESS;
 }

 _mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args)
 {
 	struct dump_info info = { 0, 0, 0, NULL };
 	struct mali_session_data *session_data;

 	MALI_DEBUG_ASSERT_POINTER(args);

 	session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
 	MALI_DEBUG_ASSERT_POINTER(session_data);

 	info.buffer_left = args->size;
 	info.buffer = (u32 *)(uintptr_t)args->buffer;

 	args->register_writes = (uintptr_t)info.buffer;
 	MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));

 	args->page_table_dump = (uintptr_t)info.buffer;
 	MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));

 	args->register_writes_size = info.register_writes_size;
 	args->page_table_dump_size = info.page_table_dump_size;

 	MALI_SUCCESS;
 }
	/*
	* Copyright (C) 2011-2016 ARM Limited. All rights reserved.
	*
	* This program is free software and is provided to you under the terms of the GNU General Public License version 2
	* as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
	*
	* A copy of the licence is included with the program, and can also be obtained from Free Software
	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
	*/

	#include "mali_kernel_common.h"
	#include "mali_osk.h"
	#include "mali_ukk.h"
	#include "mali_uk_types.h"
	#include "mali_mmu_page_directory.h"
	#include "mali_memory.h"
	#include "mali_l2_cache.h"

	static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data);

	u32 mali_allocate_empty_page(mali_io_address *virt_addr)
	{
	_mali_osk_errcode_t err;
	mali_io_address mapping;
	mali_dma_addr address;

	if (_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping)) {
	/* Allocation failed */
	MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to get table page for empty pgdir\n"));
	return 0;
	}

	MALI_DEBUG_ASSERT_POINTER(mapping);

	err = fill_page(mapping, 0);
	if (_MALI_OSK_ERR_OK != err) {
	mali_mmu_release_table_page(address, mapping);
	MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to zero page\n"));
	return 0;
	}

	*virt_addr = mapping;
	return address;
	}

	void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr)
	{
	if (MALI_INVALID_PAGE != address) {
	mali_mmu_release_table_page(address, virt_addr);
	}
	}

	_mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory,
	mali_io_address *page_directory_mapping,
	mali_dma_addr page_table, mali_io_address page_table_mapping,
	mali_dma_addr data_page, mali_io_address data_page_mapping)
	{
	_mali_osk_errcode_t err;

	err = mali_mmu_get_table_page(data_page, data_page_mapping);
	if (_MALI_OSK_ERR_OK == err) {
	err = mali_mmu_get_table_page(page_table, page_table_mapping);
	if (_MALI_OSK_ERR_OK == err) {
	err = mali_mmu_get_table_page(page_directory, page_directory_mapping);
	if (_MALI_OSK_ERR_OK == err) {
	fill_page(*data_page_mapping, 0);
	fill_page(page_table_mapping, data_page \| MALI_MMU_FLAGS_DEFAULT);
	fill_page(page_directory_mapping, page_table \| MALI_MMU_FLAGS_PRESENT);
	MALI_SUCCESS;
	}
	mali_mmu_release_table_page(page_table, page_table_mapping);
	*page_table = MALI_INVALID_PAGE;
	}
	mali_mmu_release_table_page(data_page, data_page_mapping);
	*data_page = MALI_INVALID_PAGE;
	}
	return err;
	}

	void mali_destroy_fault_flush_pages(
	mali_dma_addr page_directory, mali_io_address page_directory_mapping,
	mali_dma_addr page_table, mali_io_address page_table_mapping,
	mali_dma_addr data_page, mali_io_address data_page_mapping)
	{
	if (MALI_INVALID_PAGE != *page_directory) {
	mali_mmu_release_table_page(page_directory, page_directory_mapping);
	*page_directory = MALI_INVALID_PAGE;
	*page_directory_mapping = NULL;
	}

	if (MALI_INVALID_PAGE != *page_table) {
	mali_mmu_release_table_page(page_table, page_table_mapping);
	*page_table = MALI_INVALID_PAGE;
	*page_table_mapping = NULL;
	}

	if (MALI_INVALID_PAGE != *data_page) {
	mali_mmu_release_table_page(data_page, data_page_mapping);
	*data_page = MALI_INVALID_PAGE;
	*data_page_mapping = NULL;
	}
	}

	static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data)
	{
	int i;
	MALI_DEBUG_ASSERT_POINTER(mapping);

	for (i = 0; i < MALI_MMU_PAGE_SIZE / 4; i++) {
	_mali_osk_mem_iowrite32_relaxed(mapping, i * sizeof(u32), data);
	}
	_mali_osk_mem_barrier();
	MALI_SUCCESS;
	}

	_mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
	{
	const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
	const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
	_mali_osk_errcode_t err;
	mali_io_address pde_mapping;
	mali_dma_addr pde_phys;
	int i, page_count;
	u32 start_address;
	if (last_pde < first_pde)
	return _MALI_OSK_ERR_INVALID_ARGS;

	for (i = first_pde; i <= last_pde; i++) {
	if (0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
	i * sizeof(u32)) & MALI_MMU_FLAGS_PRESENT)) {
	/* Page table not present */
	MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
	MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]);

	err = mali_mmu_get_table_page(&pde_phys, &pde_mapping);
	if (_MALI_OSK_ERR_OK != err) {
	MALI_PRINT_ERROR(("Failed to allocate page table page.\n"));
	return err;
	}
	pagedir->page_entries_mapped[i] = pde_mapping;

	/* Update PDE, mark as present */
	_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32),
	pde_phys \| MALI_MMU_FLAGS_PRESENT);

	MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
	}

	if (first_pde == last_pde) {
	pagedir->page_entries_usage_count[i] += size / MALI_MMU_PAGE_SIZE;
	} else if (i == first_pde) {
	start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE;
	page_count = (start_address + MALI_MMU_VIRTUAL_PAGE_SIZE - mali_address) / MALI_MMU_PAGE_SIZE;
	pagedir->page_entries_usage_count[i] += page_count;
	} else if (i == last_pde) {
	start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE;
	page_count = (mali_address + size - start_address) / MALI_MMU_PAGE_SIZE;
	pagedir->page_entries_usage_count[i] += page_count;
	} else {
	pagedir->page_entries_usage_count[i] = 1024;
	}
	}
	_mali_osk_write_mem_barrier();

	return _MALI_OSK_ERR_OK;
	}

	MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size)
	{
	int i;
	const int first_pte = MALI_MMU_PTE_ENTRY(mali_address);
	const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1);

	for (i = first_pte; i <= last_pte; i++) {
	_mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0);
	}
	}

	static u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index)
	{
	return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
	index * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK);
	}


	_mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
	{
	const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
	const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
	u32 left = size;
	int i;
	mali_bool pd_changed = MALI_FALSE;
	u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */
	u32 num_pages_inv = 0;
	mali_bool invalidate_all = MALI_FALSE; /* safety mechanism in case page_entries_usage_count is unreliable */

	/* For all page directory entries in range. */
	for (i = first_pde; i <= last_pde; i++) {
	u32 size_in_pde, offset;

	MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]);
	MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]);

	/* Offset into page table, 0 if mali_address is 4MiB aligned */
	offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1));
	if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset) {
	size_in_pde = left;
	} else {
	size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset;
	}

	pagedir->page_entries_usage_count[i] -= size_in_pde / MALI_MMU_PAGE_SIZE;

	/* If entire page table is unused, free it */
	if (0 == pagedir->page_entries_usage_count[i]) {
	u32 page_phys;
	void *page_virt;
	MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n"));
	/* last reference removed, no need to zero out each PTE */

	page_phys = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i * sizeof(u32)));
	page_virt = pagedir->page_entries_mapped[i];
	pagedir->page_entries_mapped[i] = NULL;
	_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);

	mali_mmu_release_table_page(page_phys, page_virt);
	pd_changed = MALI_TRUE;
	} else {
	MALI_DEBUG_ASSERT(num_pages_inv < 2);
	if (num_pages_inv < 2) {
	pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i);
	num_pages_inv++;
	} else {
	invalidate_all = MALI_TRUE;
	}

	/* If part of the page table is still in use, zero the relevant PTEs */
	mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde);
	}

	left -= size_in_pde;
	mali_address += size_in_pde;
	}
	_mali_osk_write_mem_barrier();

	/* L2 pages invalidation */
	if (MALI_TRUE == pd_changed) {
	MALI_DEBUG_ASSERT(num_pages_inv < 3);
	if (num_pages_inv < 3) {
	pages_to_invalidate[num_pages_inv] = pagedir->page_directory;
	num_pages_inv++;
	} else {
	invalidate_all = MALI_TRUE;
	}
	}

	if (invalidate_all) {
	mali_l2_cache_invalidate_all();
	} else {
	mali_l2_cache_invalidate_all_pages(pages_to_invalidate, num_pages_inv);
	}

	MALI_SUCCESS;
	}

	struct mali_page_directory *mali_mmu_pagedir_alloc(void)
	{
	struct mali_page_directory *pagedir;
	_mali_osk_errcode_t err;
	mali_dma_addr phys;

	pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory));
	if (NULL == pagedir) {
	return NULL;
	}

	err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped);
	if (_MALI_OSK_ERR_OK != err) {
	_mali_osk_free(pagedir);
	return NULL;
	}

	pagedir->page_directory = (u32)phys;

	/* Zero page directory */
	fill_page(pagedir->page_directory_mapped, 0);

	return pagedir;
	}

	void mali_mmu_pagedir_free(struct mali_page_directory *pagedir)
	{
	const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]);
	int i;

	/* Free referenced page tables and zero PDEs. */
	for (i = 0; i < num_page_table_entries; i++) {
	if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32(
	pagedir->page_directory_mapped,
	sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT)) {
	mali_dma_addr phys = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
	i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK;
	_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);
	mali_mmu_release_table_page(phys, pagedir->page_entries_mapped[i]);
	}
	}
	_mali_osk_write_mem_barrier();

	/* Free the page directory page. */
	mali_mmu_release_table_page(pagedir->page_directory, pagedir->page_directory_mapped);

	_mali_osk_free(pagedir);
	}


	void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address,
	mali_dma_addr phys_address, u32 size, u32 permission_bits)
	{
	u32 end_address = mali_address + size;
	u32 mali_phys = (u32)phys_address;

	/* Map physical pages into MMU page tables */
	for (; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, mali_phys += MALI_MMU_PAGE_SIZE) {
	MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]);
	_mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)],
	MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32),
	mali_phys \| permission_bits);
	}
	}

	void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr)
	{
	#if defined(DEBUG)
	u32 pde_index, pte_index;
	u32 pde, pte;

	pde_index = MALI_MMU_PDE_ENTRY(fault_addr);
	pte_index = MALI_MMU_PTE_ENTRY(fault_addr);


	pde = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
	pde_index * sizeof(u32));


	if (pde & MALI_MMU_FLAGS_PRESENT) {
	u32 pte_addr = MALI_MMU_ENTRY_ADDRESS(pde);

	pte = _mali_osk_mem_ioread32(pagedir->page_entries_mapped[pde_index],
	pte_index * sizeof(u32));

	MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table present: %08x\n"
	"\t\tPTE: %08x, page %08x is %s\n",
	fault_addr, pte_addr, pte,
	MALI_MMU_ENTRY_ADDRESS(pte),
	pte & MALI_MMU_FLAGS_DEFAULT ? "rw" : "not present"));
	} else {
	MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table not present: %08x\n",
	fault_addr, pde));
	}
	#else
	MALI_IGNORE(pagedir);
	MALI_IGNORE(fault_addr);
	#endif
	}

	/* For instrumented */
	struct dump_info {
	u32 buffer_left;
	u32 register_writes_size;
	u32 page_table_dump_size;
	u32 *buffer;
	};

	static _mali_osk_errcode_t writereg(u32 where, u32 what, const char comment, struct dump_info info)
	{
	if (NULL != info) {
	info->register_writes_size += sizeof(u32) * 2; /* two 32-bit words */

	if (NULL != info->buffer) {
	/* check that we have enough space */
	if (info->buffer_left < sizeof(u32) * 2) MALI_ERROR(_MALI_OSK_ERR_NOMEM);

	*info->buffer = where;
	info->buffer++;

	*info->buffer = what;
	info->buffer++;

	info->buffer_left -= sizeof(u32) * 2;
	}
	}

	MALI_SUCCESS;
	}

	static _mali_osk_errcode_t mali_mmu_dump_page(mali_io_address page, u32 phys_addr, struct dump_info *info)
	{
	if (NULL != info) {
	/* 4096 for the page and 4 bytes for the address */
	const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4;
	const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE;
	const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4;

	info->page_table_dump_size += dump_size_in_bytes;

	if (NULL != info->buffer) {
	if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM);

	*info->buffer = phys_addr;
	info->buffer++;

	_mali_osk_memcpy(info->buffer, page, page_size_in_bytes);
	info->buffer += page_size_in_elements;

	info->buffer_left -= dump_size_in_bytes;
	}
	}

	MALI_SUCCESS;
	}

	static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory pagedir, struct dump_info info)
	{
	MALI_DEBUG_ASSERT_POINTER(pagedir);
	MALI_DEBUG_ASSERT_POINTER(info);

	if (NULL != pagedir->page_directory_mapped) {
	int i;

	MALI_CHECK_NO_ERROR(
	mali_mmu_dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info)
	);

	for (i = 0; i < 1024; i++) {
	if (NULL != pagedir->page_entries_mapped[i]) {
	MALI_CHECK_NO_ERROR(
	mali_mmu_dump_page(pagedir->page_entries_mapped[i],
	_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
	i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info)
	);
	}
	}
	}

	MALI_SUCCESS;
	}

	static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory pagedir, struct dump_info info)
	{
	MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory,
	"set the page directory address", info));
	MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info));
	MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info));
	MALI_SUCCESS;
	}

	_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args)
	{
	struct dump_info info = { 0, 0, 0, NULL };
	struct mali_session_data *session_data;

	session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
	MALI_DEBUG_ASSERT_POINTER(session_data);
	MALI_DEBUG_ASSERT_POINTER(args);

	MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
	MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
	args->size = info.register_writes_size + info.page_table_dump_size;
	MALI_SUCCESS;
	}

	_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args)
	{
	struct dump_info info = { 0, 0, 0, NULL };
	struct mali_session_data *session_data;

	MALI_DEBUG_ASSERT_POINTER(args);

	session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
	MALI_DEBUG_ASSERT_POINTER(session_data);

	info.buffer_left = args->size;
	info.buffer = (u32 *)(uintptr_t)args->buffer;

	args->register_writes = (uintptr_t)info.buffer;
	MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));

	args->page_table_dump = (uintptr_t)info.buffer;
	MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));

	args->register_writes_size = info.register_writes_size;
	args->page_table_dump_size = info.page_table_dump_size;

	MALI_SUCCESS;
	}