| /* |
| * Copyright (C) 2011-2016 ARM Limited. All rights reserved. |
| * |
| * This program is free software and is provided to you under the terms of the GNU General Public License version 2 |
| * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. |
| * |
| * A copy of the licence is included with the program, and can also be obtained from Free Software |
| * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| */ |
| |
| #include "mali_kernel_common.h" |
| #include "mali_osk.h" |
| #include "mali_ukk.h" |
| #include "mali_uk_types.h" |
| #include "mali_mmu_page_directory.h" |
| #include "mali_memory.h" |
| #include "mali_l2_cache.h" |
| |
| static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data); |
| |
| u32 mali_allocate_empty_page(mali_io_address *virt_addr) |
| { |
| _mali_osk_errcode_t err; |
| mali_io_address mapping; |
| mali_dma_addr address; |
| |
| if (_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping)) { |
| /* Allocation failed */ |
| MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to get table page for empty pgdir\n")); |
| return 0; |
| } |
| |
| MALI_DEBUG_ASSERT_POINTER(mapping); |
| |
| err = fill_page(mapping, 0); |
| if (_MALI_OSK_ERR_OK != err) { |
| mali_mmu_release_table_page(address, mapping); |
| MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to zero page\n")); |
| return 0; |
| } |
| |
| *virt_addr = mapping; |
| return address; |
| } |
| |
| void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr) |
| { |
| if (MALI_INVALID_PAGE != address) { |
| mali_mmu_release_table_page(address, virt_addr); |
| } |
| } |
| |
| _mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory, |
| mali_io_address *page_directory_mapping, |
| mali_dma_addr *page_table, mali_io_address *page_table_mapping, |
| mali_dma_addr *data_page, mali_io_address *data_page_mapping) |
| { |
| _mali_osk_errcode_t err; |
| |
| err = mali_mmu_get_table_page(data_page, data_page_mapping); |
| if (_MALI_OSK_ERR_OK == err) { |
| err = mali_mmu_get_table_page(page_table, page_table_mapping); |
| if (_MALI_OSK_ERR_OK == err) { |
| err = mali_mmu_get_table_page(page_directory, page_directory_mapping); |
| if (_MALI_OSK_ERR_OK == err) { |
| fill_page(*data_page_mapping, 0); |
| fill_page(*page_table_mapping, *data_page | MALI_MMU_FLAGS_DEFAULT); |
| fill_page(*page_directory_mapping, *page_table | MALI_MMU_FLAGS_PRESENT); |
| MALI_SUCCESS; |
| } |
| mali_mmu_release_table_page(*page_table, *page_table_mapping); |
| *page_table = MALI_INVALID_PAGE; |
| } |
| mali_mmu_release_table_page(*data_page, *data_page_mapping); |
| *data_page = MALI_INVALID_PAGE; |
| } |
| return err; |
| } |
| |
| void mali_destroy_fault_flush_pages( |
| mali_dma_addr *page_directory, mali_io_address *page_directory_mapping, |
| mali_dma_addr *page_table, mali_io_address *page_table_mapping, |
| mali_dma_addr *data_page, mali_io_address *data_page_mapping) |
| { |
| if (MALI_INVALID_PAGE != *page_directory) { |
| mali_mmu_release_table_page(*page_directory, *page_directory_mapping); |
| *page_directory = MALI_INVALID_PAGE; |
| *page_directory_mapping = NULL; |
| } |
| |
| if (MALI_INVALID_PAGE != *page_table) { |
| mali_mmu_release_table_page(*page_table, *page_table_mapping); |
| *page_table = MALI_INVALID_PAGE; |
| *page_table_mapping = NULL; |
| } |
| |
| if (MALI_INVALID_PAGE != *data_page) { |
| mali_mmu_release_table_page(*data_page, *data_page_mapping); |
| *data_page = MALI_INVALID_PAGE; |
| *data_page_mapping = NULL; |
| } |
| } |
| |
| static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data) |
| { |
| int i; |
| MALI_DEBUG_ASSERT_POINTER(mapping); |
| |
| for (i = 0; i < MALI_MMU_PAGE_SIZE / 4; i++) { |
| _mali_osk_mem_iowrite32_relaxed(mapping, i * sizeof(u32), data); |
| } |
| _mali_osk_mem_barrier(); |
| MALI_SUCCESS; |
| } |
| |
| _mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size) |
| { |
| const int first_pde = MALI_MMU_PDE_ENTRY(mali_address); |
| const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1); |
| _mali_osk_errcode_t err; |
| mali_io_address pde_mapping; |
| mali_dma_addr pde_phys; |
| int i, page_count; |
| u32 start_address; |
| if (last_pde < first_pde) |
| return _MALI_OSK_ERR_INVALID_ARGS; |
| |
| for (i = first_pde; i <= last_pde; i++) { |
| if (0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, |
| i * sizeof(u32)) & MALI_MMU_FLAGS_PRESENT)) { |
| /* Page table not present */ |
| MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]); |
| MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]); |
| |
| err = mali_mmu_get_table_page(&pde_phys, &pde_mapping); |
| if (_MALI_OSK_ERR_OK != err) { |
| MALI_PRINT_ERROR(("Failed to allocate page table page.\n")); |
| return err; |
| } |
| pagedir->page_entries_mapped[i] = pde_mapping; |
| |
| /* Update PDE, mark as present */ |
| _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), |
| pde_phys | MALI_MMU_FLAGS_PRESENT); |
| |
| MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]); |
| } |
| |
| if (first_pde == last_pde) { |
| pagedir->page_entries_usage_count[i] += size / MALI_MMU_PAGE_SIZE; |
| } else if (i == first_pde) { |
| start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE; |
| page_count = (start_address + MALI_MMU_VIRTUAL_PAGE_SIZE - mali_address) / MALI_MMU_PAGE_SIZE; |
| pagedir->page_entries_usage_count[i] += page_count; |
| } else if (i == last_pde) { |
| start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE; |
| page_count = (mali_address + size - start_address) / MALI_MMU_PAGE_SIZE; |
| pagedir->page_entries_usage_count[i] += page_count; |
| } else { |
| pagedir->page_entries_usage_count[i] = 1024; |
| } |
| } |
| _mali_osk_write_mem_barrier(); |
| |
| return _MALI_OSK_ERR_OK; |
| } |
| |
| MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size) |
| { |
| int i; |
| const int first_pte = MALI_MMU_PTE_ENTRY(mali_address); |
| const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1); |
| |
| for (i = first_pte; i <= last_pte; i++) { |
| _mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0); |
| } |
| } |
| |
| static u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index) |
| { |
| return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, |
| index * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK); |
| } |
| |
| |
| _mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size) |
| { |
| const int first_pde = MALI_MMU_PDE_ENTRY(mali_address); |
| const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1); |
| u32 left = size; |
| int i; |
| mali_bool pd_changed = MALI_FALSE; |
| u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */ |
| u32 num_pages_inv = 0; |
| mali_bool invalidate_all = MALI_FALSE; /* safety mechanism in case page_entries_usage_count is unreliable */ |
| |
| /* For all page directory entries in range. */ |
| for (i = first_pde; i <= last_pde; i++) { |
| u32 size_in_pde, offset; |
| |
| MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]); |
| MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]); |
| |
| /* Offset into page table, 0 if mali_address is 4MiB aligned */ |
| offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1)); |
| if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset) { |
| size_in_pde = left; |
| } else { |
| size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset; |
| } |
| |
| pagedir->page_entries_usage_count[i] -= size_in_pde / MALI_MMU_PAGE_SIZE; |
| |
| /* If entire page table is unused, free it */ |
| if (0 == pagedir->page_entries_usage_count[i]) { |
| u32 page_phys; |
| void *page_virt; |
| MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n")); |
| /* last reference removed, no need to zero out each PTE */ |
| |
| page_phys = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i * sizeof(u32))); |
| page_virt = pagedir->page_entries_mapped[i]; |
| pagedir->page_entries_mapped[i] = NULL; |
| _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0); |
| |
| mali_mmu_release_table_page(page_phys, page_virt); |
| pd_changed = MALI_TRUE; |
| } else { |
| MALI_DEBUG_ASSERT(num_pages_inv < 2); |
| if (num_pages_inv < 2) { |
| pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i); |
| num_pages_inv++; |
| } else { |
| invalidate_all = MALI_TRUE; |
| } |
| |
| /* If part of the page table is still in use, zero the relevant PTEs */ |
| mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde); |
| } |
| |
| left -= size_in_pde; |
| mali_address += size_in_pde; |
| } |
| _mali_osk_write_mem_barrier(); |
| |
| /* L2 pages invalidation */ |
| if (MALI_TRUE == pd_changed) { |
| MALI_DEBUG_ASSERT(num_pages_inv < 3); |
| if (num_pages_inv < 3) { |
| pages_to_invalidate[num_pages_inv] = pagedir->page_directory; |
| num_pages_inv++; |
| } else { |
| invalidate_all = MALI_TRUE; |
| } |
| } |
| |
| if (invalidate_all) { |
| mali_l2_cache_invalidate_all(); |
| } else { |
| mali_l2_cache_invalidate_all_pages(pages_to_invalidate, num_pages_inv); |
| } |
| |
| MALI_SUCCESS; |
| } |
| |
| struct mali_page_directory *mali_mmu_pagedir_alloc(void) |
| { |
| struct mali_page_directory *pagedir; |
| _mali_osk_errcode_t err; |
| mali_dma_addr phys; |
| |
| pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory)); |
| if (NULL == pagedir) { |
| return NULL; |
| } |
| |
| err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped); |
| if (_MALI_OSK_ERR_OK != err) { |
| _mali_osk_free(pagedir); |
| return NULL; |
| } |
| |
| pagedir->page_directory = (u32)phys; |
| |
| /* Zero page directory */ |
| fill_page(pagedir->page_directory_mapped, 0); |
| |
| return pagedir; |
| } |
| |
| void mali_mmu_pagedir_free(struct mali_page_directory *pagedir) |
| { |
| const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]); |
| int i; |
| |
| /* Free referenced page tables and zero PDEs. */ |
| for (i = 0; i < num_page_table_entries; i++) { |
| if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32( |
| pagedir->page_directory_mapped, |
| sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT)) { |
| mali_dma_addr phys = _mali_osk_mem_ioread32(pagedir->page_directory_mapped, |
| i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK; |
| _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0); |
| mali_mmu_release_table_page(phys, pagedir->page_entries_mapped[i]); |
| } |
| } |
| _mali_osk_write_mem_barrier(); |
| |
| /* Free the page directory page. */ |
| mali_mmu_release_table_page(pagedir->page_directory, pagedir->page_directory_mapped); |
| |
| _mali_osk_free(pagedir); |
| } |
| |
| |
| void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address, |
| mali_dma_addr phys_address, u32 size, u32 permission_bits) |
| { |
| u32 end_address = mali_address + size; |
| u32 mali_phys = (u32)phys_address; |
| |
| /* Map physical pages into MMU page tables */ |
| for (; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, mali_phys += MALI_MMU_PAGE_SIZE) { |
| MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]); |
| _mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)], |
| MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32), |
| mali_phys | permission_bits); |
| } |
| } |
| |
| void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr) |
| { |
| #if defined(DEBUG) |
| u32 pde_index, pte_index; |
| u32 pde, pte; |
| |
| pde_index = MALI_MMU_PDE_ENTRY(fault_addr); |
| pte_index = MALI_MMU_PTE_ENTRY(fault_addr); |
| |
| |
| pde = _mali_osk_mem_ioread32(pagedir->page_directory_mapped, |
| pde_index * sizeof(u32)); |
| |
| |
| if (pde & MALI_MMU_FLAGS_PRESENT) { |
| u32 pte_addr = MALI_MMU_ENTRY_ADDRESS(pde); |
| |
| pte = _mali_osk_mem_ioread32(pagedir->page_entries_mapped[pde_index], |
| pte_index * sizeof(u32)); |
| |
| MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table present: %08x\n" |
| "\t\tPTE: %08x, page %08x is %s\n", |
| fault_addr, pte_addr, pte, |
| MALI_MMU_ENTRY_ADDRESS(pte), |
| pte & MALI_MMU_FLAGS_DEFAULT ? "rw" : "not present")); |
| } else { |
| MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table not present: %08x\n", |
| fault_addr, pde)); |
| } |
| #else |
| MALI_IGNORE(pagedir); |
| MALI_IGNORE(fault_addr); |
| #endif |
| } |
| |
| /* For instrumented */ |
| struct dump_info { |
| u32 buffer_left; |
| u32 register_writes_size; |
| u32 page_table_dump_size; |
| u32 *buffer; |
| }; |
| |
| static _mali_osk_errcode_t writereg(u32 where, u32 what, const char *comment, struct dump_info *info) |
| { |
| if (NULL != info) { |
| info->register_writes_size += sizeof(u32) * 2; /* two 32-bit words */ |
| |
| if (NULL != info->buffer) { |
| /* check that we have enough space */ |
| if (info->buffer_left < sizeof(u32) * 2) MALI_ERROR(_MALI_OSK_ERR_NOMEM); |
| |
| *info->buffer = where; |
| info->buffer++; |
| |
| *info->buffer = what; |
| info->buffer++; |
| |
| info->buffer_left -= sizeof(u32) * 2; |
| } |
| } |
| |
| MALI_SUCCESS; |
| } |
| |
| static _mali_osk_errcode_t mali_mmu_dump_page(mali_io_address page, u32 phys_addr, struct dump_info *info) |
| { |
| if (NULL != info) { |
| /* 4096 for the page and 4 bytes for the address */ |
| const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4; |
| const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE; |
| const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4; |
| |
| info->page_table_dump_size += dump_size_in_bytes; |
| |
| if (NULL != info->buffer) { |
| if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM); |
| |
| *info->buffer = phys_addr; |
| info->buffer++; |
| |
| _mali_osk_memcpy(info->buffer, page, page_size_in_bytes); |
| info->buffer += page_size_in_elements; |
| |
| info->buffer_left -= dump_size_in_bytes; |
| } |
| } |
| |
| MALI_SUCCESS; |
| } |
| |
| static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory *pagedir, struct dump_info *info) |
| { |
| MALI_DEBUG_ASSERT_POINTER(pagedir); |
| MALI_DEBUG_ASSERT_POINTER(info); |
| |
| if (NULL != pagedir->page_directory_mapped) { |
| int i; |
| |
| MALI_CHECK_NO_ERROR( |
| mali_mmu_dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info) |
| ); |
| |
| for (i = 0; i < 1024; i++) { |
| if (NULL != pagedir->page_entries_mapped[i]) { |
| MALI_CHECK_NO_ERROR( |
| mali_mmu_dump_page(pagedir->page_entries_mapped[i], |
| _mali_osk_mem_ioread32(pagedir->page_directory_mapped, |
| i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info) |
| ); |
| } |
| } |
| } |
| |
| MALI_SUCCESS; |
| } |
| |
| static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory *pagedir, struct dump_info *info) |
| { |
| MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory, |
| "set the page directory address", info)); |
| MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info)); |
| MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info)); |
| MALI_SUCCESS; |
| } |
| |
| _mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args) |
| { |
| struct dump_info info = { 0, 0, 0, NULL }; |
| struct mali_session_data *session_data; |
| |
| session_data = (struct mali_session_data *)(uintptr_t)(args->ctx); |
| MALI_DEBUG_ASSERT_POINTER(session_data); |
| MALI_DEBUG_ASSERT_POINTER(args); |
| |
| MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info)); |
| MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info)); |
| args->size = info.register_writes_size + info.page_table_dump_size; |
| MALI_SUCCESS; |
| } |
| |
| _mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args) |
| { |
| struct dump_info info = { 0, 0, 0, NULL }; |
| struct mali_session_data *session_data; |
| |
| MALI_DEBUG_ASSERT_POINTER(args); |
| |
| session_data = (struct mali_session_data *)(uintptr_t)(args->ctx); |
| MALI_DEBUG_ASSERT_POINTER(session_data); |
| |
| info.buffer_left = args->size; |
| info.buffer = (u32 *)(uintptr_t)args->buffer; |
| |
| args->register_writes = (uintptr_t)info.buffer; |
| MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info)); |
| |
| args->page_table_dump = (uintptr_t)info.buffer; |
| MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info)); |
| |
| args->register_writes_size = info.register_writes_size; |
| args->page_table_dump_size = info.page_table_dump_size; |
| |
| MALI_SUCCESS; |
| } |