| /* |
| * |
| * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. |
| * |
| * This program is free software and is provided to you under the terms of the |
| * GNU General Public License version 2 as published by the Free Software |
| * Foundation, and any use by you of this program is subject to the terms |
| * of such GNU licence. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, you can access it online at |
| * http://www.gnu.org/licenses/gpl-2.0.html. |
| * |
| * SPDX-License-Identifier: GPL-2.0 |
| * |
| */ |
| |
| |
| |
| /** |
| * @file mali_kbase_mem.h |
| * Base kernel memory APIs |
| */ |
| |
| #ifndef _KBASE_MEM_H_ |
| #define _KBASE_MEM_H_ |
| |
| #ifndef _KBASE_H_ |
| #error "Don't include this file directly, use mali_kbase.h instead" |
| #endif |
| |
| #include <linux/kref.h> |
| #include "mali_base_kernel.h" |
| #include <mali_kbase_hw.h> |
| #include "mali_kbase_pm.h" |
| #include "mali_kbase_defs.h" |
| /* Required for kbase_mem_evictable_unmake */ |
| #include "mali_kbase_mem_linux.h" |
| |
| static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, |
| int pages); |
| |
| /* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */ |
| #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */ |
| |
| /* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages. |
| The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and |
| page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table |
| updates and generates duplicate page faults as the page table information used by the MMU is not valid. */ |
| #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */ |
| |
| #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */ |
| |
| /* This must always be a power of 2 */ |
| #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2) |
| #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316) |
| #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630) |
| /** |
| * A CPU mapping |
| */ |
| struct kbase_cpu_mapping { |
| struct list_head mappings_list; |
| struct kbase_mem_phy_alloc *alloc; |
| struct kbase_context *kctx; |
| struct kbase_va_region *region; |
| int count; |
| int free_on_close; |
| }; |
| |
| enum kbase_memory_type { |
| KBASE_MEM_TYPE_NATIVE, |
| KBASE_MEM_TYPE_IMPORTED_UMM, |
| KBASE_MEM_TYPE_IMPORTED_USER_BUF, |
| KBASE_MEM_TYPE_ALIAS, |
| KBASE_MEM_TYPE_RAW |
| }; |
| |
| /* internal structure, mirroring base_mem_aliasing_info, |
| * but with alloc instead of a gpu va (handle) */ |
| struct kbase_aliased { |
| struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */ |
| u64 offset; /* in pages */ |
| u64 length; /* in pages */ |
| }; |
| |
| /** |
| * @brief Physical pages tracking object properties |
| */ |
| #define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED (1u << 0) |
| #define KBASE_MEM_PHY_ALLOC_LARGE (1u << 1) |
| |
| /* struct kbase_mem_phy_alloc - Physical pages tracking object. |
| * |
| * Set up to track N pages. |
| * N not stored here, the creator holds that info. |
| * This object only tracks how many elements are actually valid (present). |
| * Changing of nents or *pages should only happen if the kbase_mem_phy_alloc |
| * is not shared with another region or client. CPU mappings are OK to |
| * exist when changing, as long as the tracked mappings objects are |
| * updated as part of the change. |
| * |
| * @kref: number of users of this alloc |
| * @gpu_mappings: count number of times mapped on the GPU |
| * @nents: 0..N |
| * @pages: N elements, only 0..nents are valid |
| * @mappings: List of CPU mappings of this physical memory allocation. |
| * @evict_node: Node used to store this allocation on the eviction list |
| * @evicted: Physical backing size when the pages where evicted |
| * @reg: Back reference to the region structure which created this |
| * allocation, or NULL if it has been freed. |
| * @type: type of buffer |
| * @permanent_map: Kernel side mapping of the alloc, shall never be |
| * referred directly. kbase_phy_alloc_mapping_get() & |
| * kbase_phy_alloc_mapping_put() pair should be used |
| * around access to the kernel-side CPU mapping so that |
| * mapping doesn't disappear whilst it is being accessed. |
| * @properties: Bitmask of properties, e.g. KBASE_MEM_PHY_ALLOC_LARGE. |
| * @group_id: A memory group ID to be passed to a platform-specific |
| * memory group manager, if present. |
| * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). |
| * @imported: member in union valid based on @a type |
| */ |
| struct kbase_mem_phy_alloc { |
| struct kref kref; |
| atomic_t gpu_mappings; |
| size_t nents; |
| struct tagged_addr *pages; |
| struct list_head mappings; |
| struct list_head evict_node; |
| size_t evicted; |
| struct kbase_va_region *reg; |
| enum kbase_memory_type type; |
| struct kbase_vmap_struct *permanent_map; |
| u8 properties; |
| u8 group_id; |
| |
| union { |
| #if defined(CONFIG_DMA_SHARED_BUFFER) |
| struct { |
| struct dma_buf *dma_buf; |
| struct dma_buf_attachment *dma_attachment; |
| unsigned int current_mapping_usage_count; |
| struct sg_table *sgt; |
| } umm; |
| #endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ |
| struct { |
| u64 stride; |
| size_t nents; |
| struct kbase_aliased *aliased; |
| } alias; |
| struct { |
| struct kbase_context *kctx; |
| /* Number of pages in this structure, including *pages. |
| * Used for kernel memory tracking. |
| */ |
| size_t nr_struct_pages; |
| } native; |
| struct kbase_alloc_import_user_buf { |
| unsigned long address; |
| unsigned long size; |
| unsigned long nr_pages; |
| struct page **pages; |
| /* top bit (1<<31) of current_mapping_usage_count |
| * specifies that this import was pinned on import |
| * See PINNED_ON_IMPORT |
| */ |
| u32 current_mapping_usage_count; |
| struct mm_struct *mm; |
| dma_addr_t *dma_addrs; |
| } user_buf; |
| } imported; |
| }; |
| |
| /* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is |
| * used to signify that a buffer was pinned when it was imported. Since the |
| * reference count is limited by the number of atoms that can be submitted at |
| * once there should be no danger of overflowing into this bit. |
| * Stealing the top bit also has the benefit that |
| * current_mapping_usage_count != 0 if and only if the buffer is mapped. |
| */ |
| #define PINNED_ON_IMPORT (1<<31) |
| |
| static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) |
| { |
| KBASE_DEBUG_ASSERT(alloc); |
| /* we only track mappings of NATIVE buffers */ |
| if (alloc->type == KBASE_MEM_TYPE_NATIVE) |
| atomic_inc(&alloc->gpu_mappings); |
| } |
| |
| static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc *alloc) |
| { |
| KBASE_DEBUG_ASSERT(alloc); |
| /* we only track mappings of NATIVE buffers */ |
| if (alloc->type == KBASE_MEM_TYPE_NATIVE) |
| if (0 > atomic_dec_return(&alloc->gpu_mappings)) { |
| pr_err("Mismatched %s:\n", __func__); |
| dump_stack(); |
| } |
| } |
| |
| /** |
| * kbase_mem_is_imported - Indicate whether a memory type is imported |
| * |
| * @type: the memory type |
| * |
| * Return: true if the memory type is imported, false otherwise |
| */ |
| static inline bool kbase_mem_is_imported(enum kbase_memory_type type) |
| { |
| return (type == KBASE_MEM_TYPE_IMPORTED_UMM) || |
| (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); |
| } |
| |
| void kbase_mem_kref_free(struct kref *kref); |
| |
| int kbase_mem_init(struct kbase_device *kbdev); |
| void kbase_mem_halt(struct kbase_device *kbdev); |
| void kbase_mem_term(struct kbase_device *kbdev); |
| |
| static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc) |
| { |
| kref_get(&alloc->kref); |
| return alloc; |
| } |
| |
| static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc) |
| { |
| kref_put(&alloc->kref, kbase_mem_kref_free); |
| return NULL; |
| } |
| |
| /** |
| * A GPU memory region, and attributes for CPU mappings. |
| */ |
| struct kbase_va_region { |
| struct rb_node rblink; |
| struct list_head link; |
| |
| struct rb_root *rbtree; /* Backlink to rb tree */ |
| |
| u64 start_pfn; /* The PFN in GPU space */ |
| size_t nr_pages; |
| /* Initial commit, for aligning the start address and correctly growing |
| * KBASE_REG_TILER_ALIGN_TOP regions */ |
| size_t initial_commit; |
| |
| /* Free region */ |
| #define KBASE_REG_FREE (1ul << 0) |
| /* CPU write access */ |
| #define KBASE_REG_CPU_WR (1ul << 1) |
| /* GPU write access */ |
| #define KBASE_REG_GPU_WR (1ul << 2) |
| /* No eXecute flag */ |
| #define KBASE_REG_GPU_NX (1ul << 3) |
| /* Is CPU cached? */ |
| #define KBASE_REG_CPU_CACHED (1ul << 4) |
| /* Is GPU cached? |
| * Some components within the GPU might only be able to access memory that is |
| * GPU cacheable. Refer to the specific GPU implementation for more details. |
| */ |
| #define KBASE_REG_GPU_CACHED (1ul << 5) |
| |
| #define KBASE_REG_GROWABLE (1ul << 6) |
| /* Can grow on pf? */ |
| #define KBASE_REG_PF_GROW (1ul << 7) |
| |
| /* Allocation doesn't straddle the 4GB boundary in GPU virtual space */ |
| #define KBASE_REG_GPU_VA_SAME_4GB_PAGE (1ul << 8) |
| |
| /* inner shareable coherency */ |
| #define KBASE_REG_SHARE_IN (1ul << 9) |
| /* inner & outer shareable coherency */ |
| #define KBASE_REG_SHARE_BOTH (1ul << 10) |
| |
| /* Space for 4 different zones */ |
| #define KBASE_REG_ZONE_MASK (3ul << 11) |
| #define KBASE_REG_ZONE(x) (((x) & 3) << 11) |
| |
| /* GPU read access */ |
| #define KBASE_REG_GPU_RD (1ul<<13) |
| /* CPU read access */ |
| #define KBASE_REG_CPU_RD (1ul<<14) |
| |
| /* Index of chosen MEMATTR for this region (0..7) */ |
| #define KBASE_REG_MEMATTR_MASK (7ul << 16) |
| #define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16) |
| #define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16) |
| |
| #define KBASE_REG_SECURE (1ul << 19) |
| |
| #define KBASE_REG_DONT_NEED (1ul << 20) |
| |
| /* Imported buffer is padded? */ |
| #define KBASE_REG_IMPORT_PAD (1ul << 21) |
| |
| /* Bit 22 is reserved. |
| * |
| * Do not remove, use the next unreserved bit for new flags */ |
| #define KBASE_REG_RESERVED_BIT_22 (1ul << 22) |
| |
| /* The top of the initial commit is aligned to extent pages. |
| * Extent must be a power of 2 */ |
| #define KBASE_REG_TILER_ALIGN_TOP (1ul << 23) |
| |
| /* Whilst this flag is set the GPU allocation is not supposed to be freed by |
| * user space. The flag will remain set for the lifetime of JIT allocations. |
| */ |
| #define KBASE_REG_NO_USER_FREE (1ul << 24) |
| |
| /* Memory has permanent kernel side mapping */ |
| #define KBASE_REG_PERMANENT_KERNEL_MAPPING (1ul << 25) |
| |
| /* GPU VA region has been freed by the userspace, but still remains allocated |
| * due to the reference held by CPU mappings created on the GPU VA region. |
| * |
| * A region with this flag set has had kbase_gpu_munmap() called on it, but can |
| * still be looked-up in the region tracker as a non-free region. Hence must |
| * not create or update any more GPU mappings on such regions because they will |
| * not be unmapped when the region is finally destroyed. |
| * |
| * Since such regions are still present in the region tracker, new allocations |
| * attempted with BASE_MEM_SAME_VA might fail if their address intersects with |
| * a region with this flag set. |
| * |
| * In addition, this flag indicates the gpu_alloc member might no longer valid |
| * e.g. in infinite cache simulation. |
| */ |
| #define KBASE_REG_VA_FREED (1ul << 26) |
| |
| #define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0) |
| |
| /* only used with 32-bit clients */ |
| /* |
| * On a 32bit platform, custom VA should be wired from 4GB |
| * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface |
| * limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference). |
| * So we put the default limit to the maximum possible on Linux and shrink |
| * it down, if required by the GPU, during initialization. |
| */ |
| |
| #define KBASE_REG_ZONE_CUSTOM_VA KBASE_REG_ZONE(1) |
| #define KBASE_REG_ZONE_CUSTOM_VA_BASE (0x100000000ULL >> PAGE_SHIFT) |
| #define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) |
| /* end 32-bit clients only */ |
| |
| /* The starting address and size of the GPU-executable zone are dynamic |
| * and depend on the platform and the number of pages requested by the |
| * user process, with an upper limit of 4 GB. |
| */ |
| #define KBASE_REG_ZONE_EXEC_VA KBASE_REG_ZONE(2) |
| #define KBASE_REG_ZONE_EXEC_VA_MAX_PAGES ((1ULL << 32) >> PAGE_SHIFT) /* 4 GB */ |
| |
| |
| unsigned long flags; |
| |
| size_t extent; /* nr of pages alloc'd on PF */ |
| |
| struct kbase_mem_phy_alloc *cpu_alloc; /* the one alloc object we mmap to the CPU when mapping this region */ |
| struct kbase_mem_phy_alloc *gpu_alloc; /* the one alloc object we mmap to the GPU when mapping this region */ |
| |
| /* List head used to store the region in the JIT allocation pool */ |
| struct list_head jit_node; |
| /* The last JIT usage ID for this region */ |
| u16 jit_usage_id; |
| /* The JIT bin this allocation came from */ |
| u8 jit_bin_id; |
| |
| int va_refcnt; /* number of users of this va */ |
| }; |
| |
| static inline bool kbase_is_region_free(struct kbase_va_region *reg) |
| { |
| return (!reg || reg->flags & KBASE_REG_FREE); |
| } |
| |
| static inline bool kbase_is_region_invalid(struct kbase_va_region *reg) |
| { |
| return (!reg || reg->flags & KBASE_REG_VA_FREED); |
| } |
| |
| static inline bool kbase_is_region_invalid_or_free(struct kbase_va_region *reg) |
| { |
| /* Possibly not all functions that find regions would be using this |
| * helper, so they need to be checked when maintaining this function. |
| */ |
| return (kbase_is_region_invalid(reg) || kbase_is_region_free(reg)); |
| } |
| |
| int kbase_remove_va_region(struct kbase_va_region *reg); |
| static inline void kbase_region_refcnt_free(struct kbase_va_region *reg) |
| { |
| /* If region was mapped then remove va region*/ |
| if (reg->start_pfn) |
| kbase_remove_va_region(reg); |
| |
| /* To detect use-after-free in debug builds */ |
| KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE); |
| kfree(reg); |
| } |
| |
| static inline struct kbase_va_region *kbase_va_region_alloc_get( |
| struct kbase_context *kctx, struct kbase_va_region *region) |
| { |
| lockdep_assert_held(&kctx->reg_lock); |
| |
| WARN_ON(!region->va_refcnt); |
| |
| /* non-atomic as kctx->reg_lock is held */ |
| region->va_refcnt++; |
| |
| return region; |
| } |
| |
| static inline struct kbase_va_region *kbase_va_region_alloc_put( |
| struct kbase_context *kctx, struct kbase_va_region *region) |
| { |
| lockdep_assert_held(&kctx->reg_lock); |
| |
| WARN_ON(region->va_refcnt <= 0); |
| WARN_ON(region->flags & KBASE_REG_FREE); |
| |
| /* non-atomic as kctx->reg_lock is held */ |
| region->va_refcnt--; |
| if (!region->va_refcnt) |
| kbase_region_refcnt_free(region); |
| |
| return NULL; |
| } |
| |
| /* Common functions */ |
| static inline struct tagged_addr *kbase_get_cpu_phy_pages( |
| struct kbase_va_region *reg) |
| { |
| KBASE_DEBUG_ASSERT(reg); |
| KBASE_DEBUG_ASSERT(reg->cpu_alloc); |
| KBASE_DEBUG_ASSERT(reg->gpu_alloc); |
| KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); |
| |
| return reg->cpu_alloc->pages; |
| } |
| |
| static inline struct tagged_addr *kbase_get_gpu_phy_pages( |
| struct kbase_va_region *reg) |
| { |
| KBASE_DEBUG_ASSERT(reg); |
| KBASE_DEBUG_ASSERT(reg->cpu_alloc); |
| KBASE_DEBUG_ASSERT(reg->gpu_alloc); |
| KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); |
| |
| return reg->gpu_alloc->pages; |
| } |
| |
| static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg) |
| { |
| KBASE_DEBUG_ASSERT(reg); |
| /* if no alloc object the backed size naturally is 0 */ |
| if (!reg->cpu_alloc) |
| return 0; |
| |
| KBASE_DEBUG_ASSERT(reg->cpu_alloc); |
| KBASE_DEBUG_ASSERT(reg->gpu_alloc); |
| KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); |
| |
| return reg->cpu_alloc->nents; |
| } |
| |
| #define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */ |
| |
| static inline struct kbase_mem_phy_alloc *kbase_alloc_create( |
| struct kbase_context *kctx, size_t nr_pages, |
| enum kbase_memory_type type, int group_id) |
| { |
| struct kbase_mem_phy_alloc *alloc; |
| size_t alloc_size = sizeof(*alloc) + sizeof(*alloc->pages) * nr_pages; |
| size_t per_page_size = sizeof(*alloc->pages); |
| |
| /* Imported pages may have page private data already in use */ |
| if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { |
| alloc_size += nr_pages * |
| sizeof(*alloc->imported.user_buf.dma_addrs); |
| per_page_size += sizeof(*alloc->imported.user_buf.dma_addrs); |
| } |
| |
| /* |
| * Prevent nr_pages*per_page_size + sizeof(*alloc) from |
| * wrapping around. |
| */ |
| if (nr_pages > ((((size_t) -1) - sizeof(*alloc)) |
| / per_page_size)) |
| return ERR_PTR(-ENOMEM); |
| |
| /* Allocate based on the size to reduce internal fragmentation of vmem */ |
| if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) |
| alloc = vzalloc(alloc_size); |
| else |
| alloc = kzalloc(alloc_size, GFP_KERNEL); |
| |
| if (!alloc) |
| return ERR_PTR(-ENOMEM); |
| |
| if (type == KBASE_MEM_TYPE_NATIVE) { |
| alloc->imported.native.nr_struct_pages = |
| (alloc_size + (PAGE_SIZE - 1)) >> PAGE_SHIFT; |
| kbase_process_page_usage_inc(kctx, |
| alloc->imported.native.nr_struct_pages); |
| } |
| |
| /* Store allocation method */ |
| if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) |
| alloc->properties |= KBASE_MEM_PHY_ALLOC_LARGE; |
| |
| kref_init(&alloc->kref); |
| atomic_set(&alloc->gpu_mappings, 0); |
| alloc->nents = 0; |
| alloc->pages = (void *)(alloc + 1); |
| INIT_LIST_HEAD(&alloc->mappings); |
| alloc->type = type; |
| alloc->group_id = group_id; |
| |
| if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) |
| alloc->imported.user_buf.dma_addrs = |
| (void *) (alloc->pages + nr_pages); |
| |
| return alloc; |
| } |
| |
| static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, |
| struct kbase_context *kctx, int group_id) |
| { |
| KBASE_DEBUG_ASSERT(reg); |
| KBASE_DEBUG_ASSERT(!reg->cpu_alloc); |
| KBASE_DEBUG_ASSERT(!reg->gpu_alloc); |
| KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE); |
| |
| reg->cpu_alloc = kbase_alloc_create(kctx, reg->nr_pages, |
| KBASE_MEM_TYPE_NATIVE, group_id); |
| if (IS_ERR(reg->cpu_alloc)) |
| return PTR_ERR(reg->cpu_alloc); |
| else if (!reg->cpu_alloc) |
| return -ENOMEM; |
| |
| reg->cpu_alloc->imported.native.kctx = kctx; |
| if (kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE) |
| && (reg->flags & KBASE_REG_CPU_CACHED)) { |
| reg->gpu_alloc = kbase_alloc_create(kctx, reg->nr_pages, |
| KBASE_MEM_TYPE_NATIVE, group_id); |
| if (IS_ERR_OR_NULL(reg->gpu_alloc)) { |
| kbase_mem_phy_alloc_put(reg->cpu_alloc); |
| return -ENOMEM; |
| } |
| reg->gpu_alloc->imported.native.kctx = kctx; |
| } else { |
| reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); |
| } |
| |
| mutex_lock(&kctx->jit_evict_lock); |
| INIT_LIST_HEAD(®->cpu_alloc->evict_node); |
| INIT_LIST_HEAD(®->gpu_alloc->evict_node); |
| mutex_unlock(&kctx->jit_evict_lock); |
| |
| reg->flags &= ~KBASE_REG_FREE; |
| |
| return 0; |
| } |
| |
| /* |
| * Max size for kbdev memory pool (in pages) |
| */ |
| #define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT) |
| |
| /* |
| * Max size for kctx memory pool (in pages) |
| */ |
| #define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) |
| |
| /* |
| * The order required for a 2MB page allocation (2^order * 4KB = 2MB) |
| */ |
| #define KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER 9 |
| |
| /* |
| * The order required for a 4KB page allocation |
| */ |
| #define KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER 0 |
| |
| /** |
| * kbase_mem_pool_config_set_max_size - Set maximum number of free pages in |
| * initial configuration of a memory pool |
| * |
| * @config: Initial configuration for a physical memory pool |
| * @max_size: Maximum number of free pages that a pool created from |
| * @config can hold |
| */ |
| static inline void kbase_mem_pool_config_set_max_size( |
| struct kbase_mem_pool_config *const config, size_t const max_size) |
| { |
| WRITE_ONCE(config->max_size, max_size); |
| } |
| |
| /** |
| * kbase_mem_pool_config_get_max_size - Get maximum number of free pages from |
| * initial configuration of a memory pool |
| * |
| * @config: Initial configuration for a physical memory pool |
| * |
| * Return: Maximum number of free pages that a pool created from @config |
| * can hold |
| */ |
| static inline size_t kbase_mem_pool_config_get_max_size( |
| const struct kbase_mem_pool_config *const config) |
| { |
| return READ_ONCE(config->max_size); |
| } |
| |
| /** |
| * kbase_mem_pool_init - Create a memory pool for a kbase device |
| * @pool: Memory pool to initialize |
| * @config: Initial configuration for the memory pool |
| * @order: Page order for physical page size (order=0=>4kB, order=9=>2MB) |
| * @group_id: A memory group ID to be passed to a platform-specific |
| * memory group manager, if present. |
| * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). |
| * @kbdev: Kbase device where memory is used |
| * @next_pool: Pointer to the next pool or NULL. |
| * |
| * Allocations from @pool are in whole pages. Each @pool has a free list where |
| * pages can be quickly allocated from. The free list is initially empty and |
| * filled whenever pages are freed back to the pool. The number of free pages |
| * in the pool will in general not exceed @max_size, but the pool may in |
| * certain corner cases grow above @max_size. |
| * |
| * If @next_pool is not NULL, we will allocate from @next_pool before going to |
| * the memory group manager. Similarly pages can spill over to @next_pool when |
| * @pool is full. Pages are zeroed before they spill over to another pool, to |
| * prevent leaking information between applications. |
| * |
| * A shrinker is registered so that Linux mm can reclaim pages from the pool as |
| * needed. |
| * |
| * Return: 0 on success, negative -errno on error |
| */ |
| int kbase_mem_pool_init(struct kbase_mem_pool *pool, |
| const struct kbase_mem_pool_config *config, |
| unsigned int order, |
| int group_id, |
| struct kbase_device *kbdev, |
| struct kbase_mem_pool *next_pool); |
| |
| /** |
| * kbase_mem_pool_term - Destroy a memory pool |
| * @pool: Memory pool to destroy |
| * |
| * Pages in the pool will spill over to @next_pool (if available) or freed to |
| * the kernel. |
| */ |
| void kbase_mem_pool_term(struct kbase_mem_pool *pool); |
| |
| /** |
| * kbase_mem_pool_alloc - Allocate a page from memory pool |
| * @pool: Memory pool to allocate from |
| * |
| * Allocations from the pool are made as follows: |
| * 1. If there are free pages in the pool, allocate a page from @pool. |
| * 2. Otherwise, if @next_pool is not NULL and has free pages, allocate a page |
| * from @next_pool. |
| * 3. Return NULL if no memory in the pool |
| * |
| * Return: Pointer to allocated page, or NULL if allocation failed. |
| * |
| * Note : This function should not be used if the pool lock is held. Use |
| * kbase_mem_pool_alloc_locked() instead. |
| */ |
| struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool); |
| |
| /** |
| * kbase_mem_pool_alloc_locked - Allocate a page from memory pool |
| * @pool: Memory pool to allocate from |
| * |
| * If there are free pages in the pool, this function allocates a page from |
| * @pool. This function does not use @next_pool. |
| * |
| * Return: Pointer to allocated page, or NULL if allocation failed. |
| * |
| * Note : Caller must hold the pool lock. |
| */ |
| struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool); |
| |
| /** |
| * kbase_mem_pool_free - Free a page to memory pool |
| * @pool: Memory pool where page should be freed |
| * @page: Page to free to the pool |
| * @dirty: Whether some of the page may be dirty in the cache. |
| * |
| * Pages are freed to the pool as follows: |
| * 1. If @pool is not full, add @page to @pool. |
| * 2. Otherwise, if @next_pool is not NULL and not full, add @page to |
| * @next_pool. |
| * 3. Finally, free @page to the kernel. |
| * |
| * Note : This function should not be used if the pool lock is held. Use |
| * kbase_mem_pool_free_locked() instead. |
| */ |
| void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page, |
| bool dirty); |
| |
| /** |
| * kbase_mem_pool_free_locked - Free a page to memory pool |
| * @pool: Memory pool where page should be freed |
| * @p: Page to free to the pool |
| * @dirty: Whether some of the page may be dirty in the cache. |
| * |
| * If @pool is not full, this function adds @page to @pool. Otherwise, @page is |
| * freed to the kernel. This function does not use @next_pool. |
| * |
| * Note : Caller must hold the pool lock. |
| */ |
| void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, |
| bool dirty); |
| |
| /** |
| * kbase_mem_pool_alloc_pages - Allocate pages from memory pool |
| * @pool: Memory pool to allocate from |
| * @nr_4k_pages: Number of pages to allocate |
| * @pages: Pointer to array where the physical address of the allocated |
| * pages will be stored. |
| * @partial_allowed: If fewer pages allocated is allowed |
| * |
| * Like kbase_mem_pool_alloc() but optimized for allocating many pages. |
| * |
| * Return: |
| * On success number of pages allocated (could be less than nr_pages if |
| * partial_allowed). |
| * On error an error code. |
| * |
| * Note : This function should not be used if the pool lock is held. Use |
| * kbase_mem_pool_alloc_pages_locked() instead. |
| * |
| * The caller must not hold vm_lock, as this could cause a deadlock if |
| * the kernel OoM killer runs. If the caller must allocate pages while holding |
| * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead. |
| */ |
| int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, |
| struct tagged_addr *pages, bool partial_allowed); |
| |
| /** |
| * kbase_mem_pool_alloc_pages_locked - Allocate pages from memory pool |
| * @pool: Memory pool to allocate from |
| * @nr_4k_pages: Number of pages to allocate |
| * @pages: Pointer to array where the physical address of the allocated |
| * pages will be stored. |
| * |
| * Like kbase_mem_pool_alloc() but optimized for allocating many pages. This |
| * version does not allocate new pages from the kernel, and therefore will never |
| * trigger the OoM killer. Therefore, it can be run while the vm_lock is held. |
| * |
| * As new pages can not be allocated, the caller must ensure there are |
| * sufficient pages in the pool. Usage of this function should look like : |
| * |
| * kbase_gpu_vm_lock(kctx); |
| * kbase_mem_pool_lock(pool) |
| * while (kbase_mem_pool_size(pool) < pages_required) { |
| * kbase_mem_pool_unlock(pool) |
| * kbase_gpu_vm_unlock(kctx); |
| * kbase_mem_pool_grow(pool) |
| * kbase_gpu_vm_lock(kctx); |
| * kbase_mem_pool_lock(pool) |
| * } |
| * kbase_mem_pool_alloc_pages_locked(pool) |
| * kbase_mem_pool_unlock(pool) |
| * Perform other processing that requires vm_lock... |
| * kbase_gpu_vm_unlock(kctx); |
| * |
| * This ensures that the pool can be grown to the required size and that the |
| * allocation can complete without another thread using the newly grown pages. |
| * |
| * Return: |
| * On success number of pages allocated. |
| * On error an error code. |
| * |
| * Note : Caller must hold the pool lock. |
| */ |
| int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, |
| size_t nr_4k_pages, struct tagged_addr *pages); |
| |
| /** |
| * kbase_mem_pool_free_pages - Free pages to memory pool |
| * @pool: Memory pool where pages should be freed |
| * @nr_pages: Number of pages to free |
| * @pages: Pointer to array holding the physical addresses of the pages to |
| * free. |
| * @dirty: Whether any pages may be dirty in the cache. |
| * @reclaimed: Whether the pages where reclaimable and thus should bypass |
| * the pool and go straight to the kernel. |
| * |
| * Like kbase_mem_pool_free() but optimized for freeing many pages. |
| */ |
| void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, |
| struct tagged_addr *pages, bool dirty, bool reclaimed); |
| |
| /** |
| * kbase_mem_pool_free_pages_locked - Free pages to memory pool |
| * @pool: Memory pool where pages should be freed |
| * @nr_pages: Number of pages to free |
| * @pages: Pointer to array holding the physical addresses of the pages to |
| * free. |
| * @dirty: Whether any pages may be dirty in the cache. |
| * @reclaimed: Whether the pages where reclaimable and thus should bypass |
| * the pool and go straight to the kernel. |
| * |
| * Like kbase_mem_pool_free() but optimized for freeing many pages. |
| */ |
| void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, |
| size_t nr_pages, struct tagged_addr *pages, bool dirty, |
| bool reclaimed); |
| |
| /** |
| * kbase_mem_pool_size - Get number of free pages in memory pool |
| * @pool: Memory pool to inspect |
| * |
| * Note: the size of the pool may in certain corner cases exceed @max_size! |
| * |
| * Return: Number of free pages in the pool |
| */ |
| static inline size_t kbase_mem_pool_size(struct kbase_mem_pool *pool) |
| { |
| return READ_ONCE(pool->cur_size); |
| } |
| |
| /** |
| * kbase_mem_pool_max_size - Get maximum number of free pages in memory pool |
| * @pool: Memory pool to inspect |
| * |
| * Return: Maximum number of free pages in the pool |
| */ |
| static inline size_t kbase_mem_pool_max_size(struct kbase_mem_pool *pool) |
| { |
| return pool->max_size; |
| } |
| |
| |
| /** |
| * kbase_mem_pool_set_max_size - Set maximum number of free pages in memory pool |
| * @pool: Memory pool to inspect |
| * @max_size: Maximum number of free pages the pool can hold |
| * |
| * If @max_size is reduced, the pool will be shrunk to adhere to the new limit. |
| * For details see kbase_mem_pool_shrink(). |
| */ |
| void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size); |
| |
| /** |
| * kbase_mem_pool_grow - Grow the pool |
| * @pool: Memory pool to grow |
| * @nr_to_grow: Number of pages to add to the pool |
| * |
| * Adds @nr_to_grow pages to the pool. Note that this may cause the pool to |
| * become larger than the maximum size specified. |
| * |
| * Returns: 0 on success, -ENOMEM if unable to allocate sufficent pages |
| */ |
| int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow); |
| |
| /** |
| * kbase_mem_pool_trim - Grow or shrink the pool to a new size |
| * @pool: Memory pool to trim |
| * @new_size: New number of pages in the pool |
| * |
| * If @new_size > @cur_size, fill the pool with new pages from the kernel, but |
| * not above the max_size for the pool. |
| * If @new_size < @cur_size, shrink the pool by freeing pages to the kernel. |
| */ |
| void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size); |
| |
| /** |
| * kbase_mem_pool_mark_dying - Mark that this pool is dying |
| * @pool: Memory pool |
| * |
| * This will cause any ongoing allocation operations (eg growing on page fault) |
| * to be terminated. |
| */ |
| void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool); |
| |
| /** |
| * kbase_mem_alloc_page - Allocate a new page for a device |
| * @pool: Memory pool to allocate a page from |
| * |
| * Most uses should use kbase_mem_pool_alloc to allocate a page. However that |
| * function can fail in the event the pool is empty. |
| * |
| * Return: A new page or NULL if no memory |
| */ |
| struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool); |
| |
| /** |
| * kbase_region_tracker_init - Initialize the region tracker data structure |
| * @kctx: kbase context |
| * |
| * Return: 0 if success, negative error code otherwise. |
| */ |
| int kbase_region_tracker_init(struct kbase_context *kctx); |
| |
| /** |
| * kbase_region_tracker_init_jit - Initialize the JIT region |
| * @kctx: kbase context |
| * @jit_va_pages: Size of the JIT region in pages |
| * @max_allocations: Maximum number of allocations allowed for the JIT region |
| * @trim_level: Trim level for the JIT region |
| * @group_id: The physical group ID from which to allocate JIT memory. |
| * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). |
| * |
| * Return: 0 if success, negative error code otherwise. |
| */ |
| int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, |
| u8 max_allocations, u8 trim_level, int group_id); |
| |
| /** |
| * kbase_region_tracker_init_exec - Initialize the EXEC_VA region |
| * @kctx: kbase context |
| * @exec_va_pages: Size of the JIT region in pages. |
| * It must not be greater than 4 GB. |
| * |
| * Return: 0 if success, negative error code otherwise. |
| */ |
| int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages); |
| |
| /** |
| * kbase_region_tracker_term - Terminate the JIT region |
| * @kctx: kbase context |
| */ |
| void kbase_region_tracker_term(struct kbase_context *kctx); |
| |
| /** |
| * kbase_region_tracker_term_rbtree - Free memory for a region tracker |
| * |
| * This will free all the regions within the region tracker |
| * |
| * @rbtree: Region tracker tree root |
| */ |
| void kbase_region_tracker_term_rbtree(struct rb_root *rbtree); |
| |
| struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address( |
| struct kbase_context *kctx, u64 gpu_addr); |
| struct kbase_va_region *kbase_find_region_enclosing_address( |
| struct rb_root *rbtree, u64 gpu_addr); |
| |
| /** |
| * @brief Check that a pointer is actually a valid region. |
| * |
| * Must be called with context lock held. |
| */ |
| struct kbase_va_region *kbase_region_tracker_find_region_base_address( |
| struct kbase_context *kctx, u64 gpu_addr); |
| struct kbase_va_region *kbase_find_region_base_address(struct rb_root *rbtree, |
| u64 gpu_addr); |
| |
| struct kbase_va_region *kbase_alloc_free_region(struct rb_root *rbtree, |
| u64 start_pfn, size_t nr_pages, int zone); |
| void kbase_free_alloced_region(struct kbase_va_region *reg); |
| int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, |
| u64 addr, size_t nr_pages, size_t align); |
| int kbase_add_va_region_rbtree(struct kbase_device *kbdev, |
| struct kbase_va_region *reg, u64 addr, size_t nr_pages, |
| size_t align); |
| |
| bool kbase_check_alloc_flags(unsigned long flags); |
| bool kbase_check_import_flags(unsigned long flags); |
| |
| /** |
| * kbase_check_alloc_sizes - check user space sizes parameters for an |
| * allocation |
| * |
| * @kctx: kbase context |
| * @flags: The flags passed from user space |
| * @va_pages: The size of the requested region, in pages. |
| * @commit_pages: Number of pages to commit initially. |
| * @extent: Number of pages to grow by on GPU page fault and/or alignment |
| * (depending on flags) |
| * |
| * Makes checks on the size parameters passed in from user space for a memory |
| * allocation call, with respect to the flags requested. |
| * |
| * Return: 0 if sizes are valid for these flags, negative error code otherwise |
| */ |
| int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, |
| u64 va_pages, u64 commit_pages, u64 extent); |
| |
| /** |
| * kbase_update_region_flags - Convert user space flags to kernel region flags |
| * |
| * @kctx: kbase context |
| * @reg: The region to update the flags on |
| * @flags: The flags passed from user space |
| * |
| * The user space flag BASE_MEM_COHERENT_SYSTEM_REQUIRED will be rejected and |
| * this function will fail if the system does not support system coherency. |
| * |
| * Return: 0 if successful, -EINVAL if the flags are not supported |
| */ |
| int kbase_update_region_flags(struct kbase_context *kctx, |
| struct kbase_va_region *reg, unsigned long flags); |
| |
| void kbase_gpu_vm_lock(struct kbase_context *kctx); |
| void kbase_gpu_vm_unlock(struct kbase_context *kctx); |
| |
| int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size); |
| |
| /** |
| * kbase_mmu_init - Initialise an object representing GPU page tables |
| * |
| * The structure should be terminated using kbase_mmu_term() |
| * |
| * @kbdev: Instance of GPU platform device, allocated from the probe method. |
| * @mmut: GPU page tables to be initialized. |
| * @kctx: Optional kbase context, may be NULL if this set of MMU tables |
| * is not associated with a context. |
| * @group_id: The physical group ID from which to allocate GPU page tables. |
| * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). |
| * |
| * Return: 0 if successful, otherwise a negative error code. |
| */ |
| int kbase_mmu_init(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, |
| struct kbase_context *kctx, int group_id); |
| /** |
| * kbase_mmu_term - Terminate an object representing GPU page tables |
| * |
| * This will free any page tables that have been allocated |
| * |
| * @kbdev: Instance of GPU platform device, allocated from the probe method. |
| * @mmut: GPU page tables to be destroyed. |
| */ |
| void kbase_mmu_term(struct kbase_device *kbdev, struct kbase_mmu_table *mmut); |
| |
| /** |
| * kbase_mmu_create_ate - Create an address translation entry |
| * |
| * @kbdev: Instance of GPU platform device, allocated from the probe method. |
| * @phy: Physical address of the page to be mapped for GPU access. |
| * @flags: Bitmask of attributes of the GPU memory region being mapped. |
| * @level: Page table level for which to build an address translation entry. |
| * @group_id: The physical memory group in which the page was allocated. |
| * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). |
| * |
| * This function creates an address translation entry to encode the physical |
| * address of a page to be mapped for access by the GPU, along with any extra |
| * attributes required for the GPU memory region. |
| * |
| * Return: An address translation entry, either in LPAE or AArch64 format |
| * (depending on the driver's configuration). |
| */ |
| u64 kbase_mmu_create_ate(struct kbase_device *kbdev, |
| struct tagged_addr phy, unsigned long flags, int level, int group_id); |
| |
| int kbase_mmu_insert_pages_no_flush(struct kbase_device *kbdev, |
| struct kbase_mmu_table *mmut, |
| const u64 start_vpfn, |
| struct tagged_addr *phys, size_t nr, |
| unsigned long flags, int group_id); |
| int kbase_mmu_insert_pages(struct kbase_device *kbdev, |
| struct kbase_mmu_table *mmut, u64 vpfn, |
| struct tagged_addr *phys, size_t nr, |
| unsigned long flags, int as_nr, int group_id); |
| int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, |
| struct tagged_addr phys, size_t nr, |
| unsigned long flags, int group_id); |
| |
| int kbase_mmu_teardown_pages(struct kbase_device *kbdev, |
| struct kbase_mmu_table *mmut, u64 vpfn, |
| size_t nr, int as_nr); |
| int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, |
| struct tagged_addr *phys, size_t nr, |
| unsigned long flags, int const group_id); |
| |
| /** |
| * @brief Register region and map it on the GPU. |
| * |
| * Call kbase_add_va_region() and map the region on the GPU. |
| */ |
| int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); |
| |
| /** |
| * @brief Remove the region from the GPU and unregister it. |
| * |
| * Must be called with context lock held. |
| */ |
| int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg); |
| |
| /** |
| * kbase_mmu_update - Configure an address space on the GPU to the specified |
| * MMU tables |
| * |
| * The caller has the following locking conditions: |
| * - It must hold kbase_device->mmu_hw_mutex |
| * - It must hold the hwaccess_lock |
| * |
| * @kbdev: Kbase device structure |
| * @mmut: The set of MMU tables to be configured on the address space |
| * @as_nr: The address space to be configured |
| */ |
| void kbase_mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, |
| int as_nr); |
| |
| /** |
| * kbase_mmu_disable() - Disable the MMU for a previously active kbase context. |
| * @kctx: Kbase context |
| * |
| * Disable and perform the required cache maintenance to remove the all |
| * data from provided kbase context from the GPU caches. |
| * |
| * The caller has the following locking conditions: |
| * - It must hold kbase_device->mmu_hw_mutex |
| * - It must hold the hwaccess_lock |
| */ |
| void kbase_mmu_disable(struct kbase_context *kctx); |
| |
| /** |
| * kbase_mmu_disable_as() - Set the MMU to unmapped mode for the specified |
| * address space. |
| * @kbdev: Kbase device |
| * @as_nr: The address space number to set to unmapped. |
| * |
| * This function must only be called during reset/power-up and it used to |
| * ensure the registers are in a known state. |
| * |
| * The caller must hold kbdev->mmu_hw_mutex. |
| */ |
| void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr); |
| |
| void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); |
| |
| /** Dump the MMU tables to a buffer |
| * |
| * This function allocates a buffer (of @c nr_pages pages) to hold a dump of the MMU tables and fills it. If the |
| * buffer is too small then the return value will be NULL. |
| * |
| * The GPU vm lock must be held when calling this function. |
| * |
| * The buffer returned should be freed with @ref vfree when it is no longer required. |
| * |
| * @param[in] kctx The kbase context to dump |
| * @param[in] nr_pages The number of pages to allocate for the buffer. |
| * |
| * @return The address of the buffer containing the MMU dump or NULL on error (including if the @c nr_pages is too |
| * small) |
| */ |
| void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); |
| |
| /** |
| * kbase_sync_now - Perform cache maintenance on a memory region |
| * |
| * @kctx: The kbase context of the region |
| * @sset: A syncset structure describing the region and direction of the |
| * synchronisation required |
| * |
| * Return: 0 on success or error code |
| */ |
| int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset); |
| void kbase_sync_single(struct kbase_context *kctx, struct tagged_addr cpu_pa, |
| struct tagged_addr gpu_pa, off_t offset, size_t size, |
| enum kbase_sync_type sync_fn); |
| void kbase_pre_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); |
| void kbase_post_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); |
| |
| /* OS specific functions */ |
| int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr); |
| int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg); |
| void kbase_os_mem_map_lock(struct kbase_context *kctx); |
| void kbase_os_mem_map_unlock(struct kbase_context *kctx); |
| |
| /** |
| * @brief Update the memory allocation counters for the current process |
| * |
| * OS specific call to updates the current memory allocation counters for the current process with |
| * the supplied delta. |
| * |
| * @param[in] kctx The kbase context |
| * @param[in] pages The desired delta to apply to the memory usage counters. |
| */ |
| |
| void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages); |
| |
| /** |
| * @brief Add to the memory allocation counters for the current process |
| * |
| * OS specific call to add to the current memory allocation counters for the current process by |
| * the supplied amount. |
| * |
| * @param[in] kctx The kernel base context used for the allocation. |
| * @param[in] pages The desired delta to apply to the memory usage counters. |
| */ |
| |
| static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages) |
| { |
| kbasep_os_process_page_usage_update(kctx, pages); |
| } |
| |
| /** |
| * @brief Subtract from the memory allocation counters for the current process |
| * |
| * OS specific call to subtract from the current memory allocation counters for the current process by |
| * the supplied amount. |
| * |
| * @param[in] kctx The kernel base context used for the allocation. |
| * @param[in] pages The desired delta to apply to the memory usage counters. |
| */ |
| |
| static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages) |
| { |
| kbasep_os_process_page_usage_update(kctx, 0 - pages); |
| } |
| |
| /** |
| * kbasep_find_enclosing_cpu_mapping_offset() - Find the offset of the CPU |
| * mapping of a memory allocation containing a given address range |
| * |
| * Searches for a CPU mapping of any part of any region that fully encloses the |
| * CPU virtual address range specified by @uaddr and @size. Returns a failure |
| * indication if only part of the address range lies within a CPU mapping. |
| * |
| * @kctx: The kernel base context used for the allocation. |
| * @uaddr: Start of the CPU virtual address range. |
| * @size: Size of the CPU virtual address range (in bytes). |
| * @offset: The offset from the start of the allocation to the specified CPU |
| * virtual address. |
| * |
| * Return: 0 if offset was obtained successfully. Error code otherwise. |
| */ |
| int kbasep_find_enclosing_cpu_mapping_offset( |
| struct kbase_context *kctx, |
| unsigned long uaddr, size_t size, u64 *offset); |
| |
| /** |
| * kbasep_find_enclosing_gpu_mapping_start_and_offset() - Find the address of |
| * the start of GPU virtual memory region which encloses @gpu_addr for the |
| * @size length in bytes |
| * |
| * Searches for the memory region in GPU virtual memory space which contains |
| * the region defined by the @gpu_addr and @size, where @gpu_addr is the |
| * beginning and @size the length in bytes of the provided region. If found, |
| * the location of the start address of the GPU virtual memory region is |
| * passed in @start pointer and the location of the offset of the region into |
| * the GPU virtual memory region is passed in @offset pointer. |
| * |
| * @kctx: The kernel base context within which the memory is searched. |
| * @gpu_addr: GPU virtual address for which the region is sought; defines |
| * the beginning of the provided region. |
| * @size: The length (in bytes) of the provided region for which the |
| * GPU virtual memory region is sought. |
| * @start: Pointer to the location where the address of the start of |
| * the found GPU virtual memory region is. |
| * @offset: Pointer to the location where the offset of @gpu_addr into |
| * the found GPU virtual memory region is. |
| */ |
| int kbasep_find_enclosing_gpu_mapping_start_and_offset( |
| struct kbase_context *kctx, |
| u64 gpu_addr, size_t size, u64 *start, u64 *offset); |
| |
| enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer); |
| void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); |
| void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); |
| |
| /** |
| * kbase_alloc_phy_pages_helper - Allocates physical pages. |
| * @alloc: allocation object to add pages to |
| * @nr_pages_requested: number of physical pages to allocate |
| * |
| * Allocates \a nr_pages_requested and updates the alloc object. |
| * |
| * Return: 0 if all pages have been successfully allocated. Error code otherwise |
| * |
| * Note : The caller must not hold vm_lock, as this could cause a deadlock if |
| * the kernel OoM killer runs. If the caller must allocate pages while holding |
| * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead. |
| * |
| * This function cannot be used from interrupt context |
| */ |
| int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, |
| size_t nr_pages_requested); |
| |
| /** |
| * kbase_alloc_phy_pages_helper_locked - Allocates physical pages. |
| * @alloc: allocation object to add pages to |
| * @pool: Memory pool to allocate from |
| * @nr_pages_requested: number of physical pages to allocate |
| * @prealloc_sa: Information about the partial allocation if the amount |
| * of memory requested is not a multiple of 2MB. One |
| * instance of struct kbase_sub_alloc must be allocated by |
| * the caller iff CONFIG_MALI_2MB_ALLOC is enabled. |
| * |
| * Allocates \a nr_pages_requested and updates the alloc object. This function |
| * does not allocate new pages from the kernel, and therefore will never trigger |
| * the OoM killer. Therefore, it can be run while the vm_lock is held. |
| * |
| * As new pages can not be allocated, the caller must ensure there are |
| * sufficient pages in the pool. Usage of this function should look like : |
| * |
| * kbase_gpu_vm_lock(kctx); |
| * kbase_mem_pool_lock(pool) |
| * while (kbase_mem_pool_size(pool) < pages_required) { |
| * kbase_mem_pool_unlock(pool) |
| * kbase_gpu_vm_unlock(kctx); |
| * kbase_mem_pool_grow(pool) |
| * kbase_gpu_vm_lock(kctx); |
| * kbase_mem_pool_lock(pool) |
| * } |
| * kbase_alloc_phy_pages_helper_locked(pool) |
| * kbase_mem_pool_unlock(pool) |
| * Perform other processing that requires vm_lock... |
| * kbase_gpu_vm_unlock(kctx); |
| * |
| * This ensures that the pool can be grown to the required size and that the |
| * allocation can complete without another thread using the newly grown pages. |
| * |
| * If CONFIG_MALI_2MB_ALLOC is defined and the allocation is >= 2MB, then |
| * @pool must be alloc->imported.native.kctx->lp_mem_pool. Otherwise it must be |
| * alloc->imported.native.kctx->mem_pool. |
| * @prealloc_sa is used to manage the non-2MB sub-allocation. It has to be |
| * pre-allocated because we must not sleep (due to the usage of kmalloc()) |
| * whilst holding pool->pool_lock. |
| * @prealloc_sa shall be set to NULL if it has been consumed by this function |
| * to indicate that the caller must not free it. |
| * |
| * Return: Pointer to array of allocated pages. NULL on failure. |
| * |
| * Note : Caller must hold pool->pool_lock |
| */ |
| struct tagged_addr *kbase_alloc_phy_pages_helper_locked( |
| struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, |
| size_t nr_pages_requested, |
| struct kbase_sub_alloc **prealloc_sa); |
| |
| /** |
| * @brief Free physical pages. |
| * |
| * Frees \a nr_pages and updates the alloc object. |
| * |
| * @param[in] alloc allocation object to free pages from |
| * @param[in] nr_pages_to_free number of physical pages to free |
| * |
| * Return: 0 on success, otherwise a negative error code |
| */ |
| int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free); |
| |
| /** |
| * kbase_free_phy_pages_helper_locked - Free pages allocated with |
| * kbase_alloc_phy_pages_helper_locked() |
| * @alloc: Allocation object to free pages from |
| * @pool: Memory pool to return freed pages to |
| * @pages: Pages allocated by kbase_alloc_phy_pages_helper_locked() |
| * @nr_pages_to_free: Number of physical pages to free |
| * |
| * This function atomically frees pages allocated with |
| * kbase_alloc_phy_pages_helper_locked(). @pages is the pointer to the page |
| * array that is returned by that function. @pool must be the pool that the |
| * pages were originally allocated from. |
| * |
| * If the mem_pool has been unlocked since the allocation then |
| * kbase_free_phy_pages_helper() should be used instead. |
| */ |
| void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, |
| struct kbase_mem_pool *pool, struct tagged_addr *pages, |
| size_t nr_pages_to_free); |
| |
| static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) |
| { |
| SetPagePrivate(p); |
| if (sizeof(dma_addr_t) > sizeof(p->private)) { |
| /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the |
| * private field stays the same. So we have to be clever and |
| * use the fact that we only store DMA addresses of whole pages, |
| * so the low bits should be zero */ |
| KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1))); |
| set_page_private(p, dma_addr >> PAGE_SHIFT); |
| } else { |
| set_page_private(p, dma_addr); |
| } |
| } |
| |
| static inline dma_addr_t kbase_dma_addr(struct page *p) |
| { |
| if (sizeof(dma_addr_t) > sizeof(p->private)) |
| return ((dma_addr_t)page_private(p)) << PAGE_SHIFT; |
| |
| return (dma_addr_t)page_private(p); |
| } |
| |
| static inline void kbase_clear_dma_addr(struct page *p) |
| { |
| ClearPagePrivate(p); |
| } |
| |
| /** |
| * kbase_mmu_interrupt_process - Process a bus or page fault. |
| * @kbdev The kbase_device the fault happened on |
| * @kctx The kbase_context for the faulting address space if one was found. |
| * @as The address space that has the fault |
| * @fault Data relating to the fault |
| * |
| * This function will process a fault on a specific address space |
| */ |
| void kbase_mmu_interrupt_process(struct kbase_device *kbdev, |
| struct kbase_context *kctx, struct kbase_as *as, |
| struct kbase_fault *fault); |
| |
| |
| /** |
| * @brief Process a page fault. |
| * |
| * @param[in] data work_struct passed by queue_work() |
| */ |
| void page_fault_worker(struct work_struct *data); |
| |
| /** |
| * @brief Process a bus fault. |
| * |
| * @param[in] data work_struct passed by queue_work() |
| */ |
| void bus_fault_worker(struct work_struct *data); |
| |
| /** |
| * @brief Flush MMU workqueues. |
| * |
| * This function will cause any outstanding page or bus faults to be processed. |
| * It should be called prior to powering off the GPU. |
| * |
| * @param[in] kbdev Device pointer |
| */ |
| void kbase_flush_mmu_wqs(struct kbase_device *kbdev); |
| |
| /** |
| * kbase_sync_single_for_device - update physical memory and give GPU ownership |
| * @kbdev: Device pointer |
| * @handle: DMA address of region |
| * @size: Size of region to sync |
| * @dir: DMA data direction |
| */ |
| |
| void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, |
| size_t size, enum dma_data_direction dir); |
| |
| /** |
| * kbase_sync_single_for_cpu - update physical memory and give CPU ownership |
| * @kbdev: Device pointer |
| * @handle: DMA address of region |
| * @size: Size of region to sync |
| * @dir: DMA data direction |
| */ |
| |
| void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, |
| size_t size, enum dma_data_direction dir); |
| |
| #ifdef CONFIG_DEBUG_FS |
| /** |
| * kbase_jit_debugfs_init - Add per context debugfs entry for JIT. |
| * @kctx: kbase context |
| */ |
| void kbase_jit_debugfs_init(struct kbase_context *kctx); |
| #endif /* CONFIG_DEBUG_FS */ |
| |
| /** |
| * kbase_jit_init - Initialize the JIT memory pool management |
| * @kctx: kbase context |
| * |
| * Returns zero on success or negative error number on failure. |
| */ |
| int kbase_jit_init(struct kbase_context *kctx); |
| |
| /** |
| * kbase_jit_allocate - Allocate JIT memory |
| * @kctx: kbase context |
| * @info: JIT allocation information |
| * |
| * Return: JIT allocation on success or NULL on failure. |
| */ |
| struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, |
| struct base_jit_alloc_info *info); |
| |
| /** |
| * kbase_jit_free - Free a JIT allocation |
| * @kctx: kbase context |
| * @reg: JIT allocation |
| * |
| * Frees a JIT allocation and places it into the free pool for later reuse. |
| */ |
| void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg); |
| |
| /** |
| * kbase_jit_backing_lost - Inform JIT that an allocation has lost backing |
| * @reg: JIT allocation |
| */ |
| void kbase_jit_backing_lost(struct kbase_va_region *reg); |
| |
| /** |
| * kbase_jit_evict - Evict a JIT allocation from the pool |
| * @kctx: kbase context |
| * |
| * Evict the least recently used JIT allocation from the pool. This can be |
| * required if normal VA allocations are failing due to VA exhaustion. |
| * |
| * Return: True if a JIT allocation was freed, false otherwise. |
| */ |
| bool kbase_jit_evict(struct kbase_context *kctx); |
| |
| /** |
| * kbase_jit_term - Terminate the JIT memory pool management |
| * @kctx: kbase context |
| */ |
| void kbase_jit_term(struct kbase_context *kctx); |
| |
| /** |
| * kbase_has_exec_va_zone - EXEC_VA zone predicate |
| * |
| * Determine whether an EXEC_VA zone has been created for the GPU address space |
| * of the given kbase context. |
| * |
| * @kctx: kbase context |
| * |
| * Return: True if the kbase context has an EXEC_VA zone. |
| */ |
| bool kbase_has_exec_va_zone(struct kbase_context *kctx); |
| |
| /** |
| * kbase_map_external_resource - Map an external resource to the GPU. |
| * @kctx: kbase context. |
| * @reg: The region to map. |
| * @locked_mm: The mm_struct which has been locked for this operation. |
| * |
| * Return: The physical allocation which backs the region on success or NULL |
| * on failure. |
| */ |
| struct kbase_mem_phy_alloc *kbase_map_external_resource( |
| struct kbase_context *kctx, struct kbase_va_region *reg, |
| struct mm_struct *locked_mm); |
| |
| /** |
| * kbase_unmap_external_resource - Unmap an external resource from the GPU. |
| * @kctx: kbase context. |
| * @reg: The region to unmap or NULL if it has already been released. |
| * @alloc: The physical allocation being unmapped. |
| */ |
| void kbase_unmap_external_resource(struct kbase_context *kctx, |
| struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); |
| |
| |
| /** |
| * kbase_jd_user_buf_pin_pages - Pin the pages of a user buffer. |
| * @kctx: kbase context. |
| * @reg: The region associated with the imported user buffer. |
| * |
| * To successfully pin the pages for a user buffer the current mm_struct must |
| * be the same as the mm_struct of the user buffer. After successfully pinning |
| * the pages further calls to this function succeed without doing work. |
| * |
| * Return: zero on success or negative number on failure. |
| */ |
| int kbase_jd_user_buf_pin_pages(struct kbase_context *kctx, |
| struct kbase_va_region *reg); |
| |
| /** |
| * kbase_sticky_resource_init - Initialize sticky resource management. |
| * @kctx: kbase context |
| * |
| * Returns zero on success or negative error number on failure. |
| */ |
| int kbase_sticky_resource_init(struct kbase_context *kctx); |
| |
| /** |
| * kbase_sticky_resource_acquire - Acquire a reference on a sticky resource. |
| * @kctx: kbase context. |
| * @gpu_addr: The GPU address of the external resource. |
| * |
| * Return: The metadata object which represents the binding between the |
| * external resource and the kbase context on success or NULL on failure. |
| */ |
| struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( |
| struct kbase_context *kctx, u64 gpu_addr); |
| |
| /** |
| * kbase_sticky_resource_release - Release a reference on a sticky resource. |
| * @kctx: kbase context. |
| * @meta: Binding metadata. |
| * @gpu_addr: GPU address of the external resource. |
| * |
| * If meta is NULL then gpu_addr will be used to scan the metadata list and |
| * find the matching metadata (if any), otherwise the provided meta will be |
| * used and gpu_addr will be ignored. |
| * |
| * Return: True if the release found the metadata and the reference was dropped. |
| */ |
| bool kbase_sticky_resource_release(struct kbase_context *kctx, |
| struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); |
| |
| /** |
| * kbase_sticky_resource_term - Terminate sticky resource management. |
| * @kctx: kbase context |
| */ |
| void kbase_sticky_resource_term(struct kbase_context *kctx); |
| |
| /** |
| * kbase_mem_pool_lock - Lock a memory pool |
| * @pool: Memory pool to lock |
| */ |
| static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool) |
| { |
| spin_lock(&pool->pool_lock); |
| } |
| |
| /** |
| * kbase_mem_pool_lock - Release a memory pool |
| * @pool: Memory pool to lock |
| */ |
| static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) |
| { |
| spin_unlock(&pool->pool_lock); |
| } |
| |
| /** |
| * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable. |
| * @alloc: The physical allocation |
| */ |
| void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc); |
| |
| |
| #if defined(CONFIG_DMA_SHARED_BUFFER) |
| /** |
| * kbase_mem_umm_map - Map dma-buf |
| * @kctx: Pointer to the kbase context |
| * @reg: Pointer to the region of the imported dma-buf to map |
| * |
| * Map a dma-buf on the GPU. The mappings are reference counted. |
| * |
| * Returns 0 on success, or a negative error code. |
| */ |
| int kbase_mem_umm_map(struct kbase_context *kctx, |
| struct kbase_va_region *reg); |
| |
| /** |
| * kbase_mem_umm_unmap - Unmap dma-buf |
| * @kctx: Pointer to the kbase context |
| * @reg: Pointer to the region of the imported dma-buf to unmap |
| * @alloc: Pointer to the alloc to release |
| * |
| * Unmap a dma-buf from the GPU. The mappings are reference counted. |
| * |
| * @reg must be the original region with GPU mapping of @alloc; or NULL. If |
| * @reg is NULL, or doesn't match @alloc, the GPU page table entries matching |
| * @reg will not be updated. |
| * |
| * @alloc must be a valid physical allocation of type |
| * KBASE_MEM_TYPE_IMPORTED_UMM that was previously mapped by |
| * kbase_mem_umm_map(). The dma-buf attachment referenced by @alloc will |
| * release it's mapping reference, and if the refcount reaches 0, also be be |
| * unmapped, regardless of the value of @reg. |
| */ |
| void kbase_mem_umm_unmap(struct kbase_context *kctx, |
| struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); |
| |
| /** |
| * kbase_mem_do_sync_imported - Sync caches for imported memory |
| * @kctx: Pointer to the kbase context |
| * @reg: Pointer to the region with imported memory to sync |
| * @sync_fn: The type of sync operation to perform |
| * |
| * Sync CPU caches for supported (currently only dma-buf (UMM)) memory. |
| * Attempting to sync unsupported imported memory types will result in an error |
| * code, -EINVAL. |
| * |
| * Return: 0 on success, or a negative error code. |
| */ |
| int kbase_mem_do_sync_imported(struct kbase_context *kctx, |
| struct kbase_va_region *reg, enum kbase_sync_type sync_fn); |
| #endif /* CONFIG_DMA_SHARED_BUFFER */ |
| |
| #endif /* _KBASE_MEM_H_ */ |