| // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note |
| /* |
| * |
| * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. |
| * |
| * This program is free software and is provided to you under the terms of the |
| * GNU General Public License version 2 as published by the Free Software |
| * Foundation, and any use by you of this program is subject to the terms |
| * of such GNU license. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, you can access it online at |
| * http://www.gnu.org/licenses/gpl-2.0.html. |
| * |
| */ |
| |
| #include <tl/mali_kbase_tracepoints.h> |
| |
| #include "mali_kbase_csf_tiler_heap.h" |
| #include "mali_kbase_csf_tiler_heap_def.h" |
| #include "mali_kbase_csf_heap_context_alloc.h" |
| |
| /** |
| * encode_chunk_ptr - Encode the address and size of a chunk as an integer. |
| * |
| * The size and address of the next chunk in a list are packed into a single |
| * 64-bit value for storage in a chunk's header. This function returns that |
| * value. |
| * |
| * @chunk_size: Size of a tiler heap chunk, in bytes. |
| * @chunk_addr: GPU virtual address of the same tiler heap chunk. |
| * |
| * Return: Next chunk pointer suitable for writing into a chunk header. |
| */ |
| static u64 encode_chunk_ptr(u32 const chunk_size, u64 const chunk_addr) |
| { |
| u64 encoded_size, encoded_addr; |
| |
| WARN_ON(chunk_size & ~CHUNK_SIZE_MASK); |
| WARN_ON(chunk_addr & ~CHUNK_ADDR_MASK); |
| |
| encoded_size = |
| (u64)(chunk_size >> CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT) << |
| CHUNK_HDR_NEXT_SIZE_POS; |
| |
| encoded_addr = |
| (chunk_addr >> CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT) << |
| CHUNK_HDR_NEXT_ADDR_POS; |
| |
| return (encoded_size & CHUNK_HDR_NEXT_SIZE_MASK) | |
| (encoded_addr & CHUNK_HDR_NEXT_ADDR_MASK); |
| } |
| |
| /** |
| * get_last_chunk - Get the last chunk of a tiler heap |
| * |
| * @heap: Pointer to the tiler heap. |
| * |
| * Return: The address of the most recently-linked chunk, or NULL if none. |
| */ |
| static struct kbase_csf_tiler_heap_chunk *get_last_chunk( |
| struct kbase_csf_tiler_heap *const heap) |
| { |
| lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); |
| |
| if (list_empty(&heap->chunks_list)) |
| return NULL; |
| |
| return list_last_entry(&heap->chunks_list, |
| struct kbase_csf_tiler_heap_chunk, link); |
| } |
| |
| /** |
| * link_chunk - Link a chunk into a tiler heap |
| * |
| * Unless the @chunk is the first in the kernel's list of chunks belonging to |
| * a given tiler heap, this function stores the size and address of the @chunk |
| * in the header of the preceding chunk. This requires the GPU memory region |
| * containing the header to be be mapped temporarily, which can fail. |
| * |
| * @heap: Pointer to the tiler heap. |
| * @chunk: Pointer to the heap chunk to be linked. |
| * |
| * Return: 0 if successful or a negative error code on failure. |
| */ |
| static int link_chunk(struct kbase_csf_tiler_heap *const heap, |
| struct kbase_csf_tiler_heap_chunk *const chunk) |
| { |
| struct kbase_csf_tiler_heap_chunk *const prev = get_last_chunk(heap); |
| |
| if (prev) { |
| struct kbase_context *const kctx = heap->kctx; |
| struct kbase_vmap_struct map; |
| u64 *const prev_hdr = kbase_vmap_prot(kctx, prev->gpu_va, |
| sizeof(*prev_hdr), KBASE_REG_CPU_WR, &map); |
| |
| if (unlikely(!prev_hdr)) { |
| dev_err(kctx->kbdev->dev, |
| "Failed to map tiler heap chunk 0x%llX\n", |
| prev->gpu_va); |
| return -ENOMEM; |
| } |
| |
| *prev_hdr = encode_chunk_ptr(heap->chunk_size, chunk->gpu_va); |
| kbase_vunmap(kctx, &map); |
| |
| dev_dbg(kctx->kbdev->dev, |
| "Linked tiler heap chunks, 0x%llX -> 0x%llX\n", |
| prev->gpu_va, chunk->gpu_va); |
| } |
| |
| return 0; |
| } |
| |
| /** |
| * init_chunk - Initialize and link a tiler heap chunk |
| * |
| * Zero-initialize a new chunk's header (including its pointer to the next |
| * chunk, which doesn't exist yet) and then update the previous chunk's |
| * header to link the new chunk into the chunk list. |
| * |
| * @heap: Pointer to the tiler heap. |
| * @chunk: Pointer to the heap chunk to be initialized and linked. |
| * @link_with_prev: Flag to indicate if the new chunk needs to be linked with |
| * the previously allocated chunk. |
| * |
| * Return: 0 if successful or a negative error code on failure. |
| */ |
| static int init_chunk(struct kbase_csf_tiler_heap *const heap, |
| struct kbase_csf_tiler_heap_chunk *const chunk, bool link_with_prev) |
| { |
| struct kbase_vmap_struct map; |
| struct u64 *chunk_hdr = NULL; |
| struct kbase_context *const kctx = heap->kctx; |
| |
| if (unlikely(chunk->gpu_va & ~CHUNK_ADDR_MASK)) { |
| dev_err(kctx->kbdev->dev, |
| "Tiler heap chunk address is unusable\n"); |
| return -EINVAL; |
| } |
| |
| chunk_hdr = kbase_vmap_prot(kctx, |
| chunk->gpu_va, CHUNK_HDR_SIZE, KBASE_REG_CPU_WR, &map); |
| |
| if (unlikely(!chunk_hdr)) { |
| dev_err(kctx->kbdev->dev, |
| "Failed to map a tiler heap chunk header\n"); |
| return -ENOMEM; |
| } |
| |
| memset(chunk_hdr, 0, CHUNK_HDR_SIZE); |
| kbase_vunmap(kctx, &map); |
| |
| if (link_with_prev) |
| return link_chunk(heap, chunk); |
| else |
| return 0; |
| } |
| |
| /** |
| * create_chunk - Create a tiler heap chunk |
| * |
| * This function allocates a chunk of memory for a tiler heap and adds it to |
| * the end of the list of chunks associated with that heap. The size of the |
| * chunk is not a parameter because it is configured per-heap not per-chunk. |
| * |
| * @heap: Pointer to the tiler heap for which to allocate memory. |
| * @link_with_prev: Flag to indicate if the chunk to be allocated needs to be |
| * linked with the previously allocated chunk. |
| * |
| * Return: 0 if successful or a negative error code on failure. |
| */ |
| static int create_chunk(struct kbase_csf_tiler_heap *const heap, |
| bool link_with_prev) |
| { |
| int err = 0; |
| struct kbase_context *const kctx = heap->kctx; |
| u64 nr_pages = PFN_UP(heap->chunk_size); |
| u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | |
| BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE | |
| BASE_MEM_COHERENT_LOCAL; |
| struct kbase_csf_tiler_heap_chunk *chunk = NULL; |
| |
| flags |= base_mem_group_id_set(kctx->jit_group_id); |
| |
| #if defined(CONFIG_MALI_DEBUG) || defined(CONFIG_MALI_VECTOR_DUMP) |
| flags |= BASE_MEM_PROT_CPU_RD; |
| #endif |
| |
| lockdep_assert_held(&kctx->csf.tiler_heaps.lock); |
| |
| chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); |
| if (unlikely(!chunk)) { |
| dev_err(kctx->kbdev->dev, |
| "No kernel memory for a new tiler heap chunk\n"); |
| return -ENOMEM; |
| } |
| |
| /* Allocate GPU memory for the new chunk. */ |
| INIT_LIST_HEAD(&chunk->link); |
| chunk->region = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, |
| &flags, &chunk->gpu_va); |
| |
| if (unlikely(!chunk->region)) { |
| dev_err(kctx->kbdev->dev, |
| "Failed to allocate a tiler heap chunk\n"); |
| err = -ENOMEM; |
| } else { |
| err = init_chunk(heap, chunk, link_with_prev); |
| if (unlikely(err)) { |
| kbase_gpu_vm_lock(kctx); |
| chunk->region->flags &= ~KBASE_REG_NO_USER_FREE; |
| kbase_mem_free_region(kctx, chunk->region); |
| kbase_gpu_vm_unlock(kctx); |
| } |
| } |
| |
| if (unlikely(err)) { |
| kfree(chunk); |
| } else { |
| list_add_tail(&chunk->link, &heap->chunks_list); |
| heap->chunk_count++; |
| |
| dev_dbg(kctx->kbdev->dev, "Created tiler heap chunk 0x%llX\n", |
| chunk->gpu_va); |
| } |
| |
| return err; |
| } |
| |
| /** |
| * delete_chunk - Delete a tiler heap chunk |
| * |
| * This function frees a tiler heap chunk previously allocated by @create_chunk |
| * and removes it from the list of chunks associated with the heap. |
| * |
| * WARNING: The deleted chunk is not unlinked from the list of chunks used by |
| * the GPU, therefore it is only safe to use this function when |
| * deleting a heap. |
| * |
| * @heap: Pointer to the tiler heap for which @chunk was allocated. |
| * @chunk: Pointer to a chunk to be deleted. |
| */ |
| static void delete_chunk(struct kbase_csf_tiler_heap *const heap, |
| struct kbase_csf_tiler_heap_chunk *const chunk) |
| { |
| struct kbase_context *const kctx = heap->kctx; |
| |
| lockdep_assert_held(&kctx->csf.tiler_heaps.lock); |
| |
| kbase_gpu_vm_lock(kctx); |
| chunk->region->flags &= ~KBASE_REG_NO_USER_FREE; |
| kbase_mem_free_region(kctx, chunk->region); |
| kbase_gpu_vm_unlock(kctx); |
| list_del(&chunk->link); |
| heap->chunk_count--; |
| kfree(chunk); |
| } |
| |
| /** |
| * delete_all_chunks - Delete all chunks belonging to a tiler heap |
| * |
| * This function empties the list of chunks associated with a tiler heap by |
| * freeing all chunks previously allocated by @create_chunk. |
| * |
| * @heap: Pointer to a tiler heap. |
| */ |
| static void delete_all_chunks(struct kbase_csf_tiler_heap *heap) |
| { |
| struct list_head *entry = NULL, *tmp = NULL; |
| struct kbase_context *const kctx = heap->kctx; |
| |
| lockdep_assert_held(&kctx->csf.tiler_heaps.lock); |
| |
| list_for_each_safe(entry, tmp, &heap->chunks_list) { |
| struct kbase_csf_tiler_heap_chunk *chunk = list_entry( |
| entry, struct kbase_csf_tiler_heap_chunk, link); |
| |
| delete_chunk(heap, chunk); |
| } |
| } |
| |
| /** |
| * create_initial_chunks - Create the initial list of chunks for a tiler heap |
| * |
| * This function allocates a given number of chunks for a tiler heap and |
| * adds them to the list of chunks associated with that heap. |
| * |
| * @heap: Pointer to the tiler heap for which to allocate memory. |
| * @nchunks: Number of chunks to create. |
| * |
| * Return: 0 if successful or a negative error code on failure. |
| */ |
| static int create_initial_chunks(struct kbase_csf_tiler_heap *const heap, |
| u32 const nchunks) |
| { |
| int err = 0; |
| u32 i; |
| |
| for (i = 0; (i < nchunks) && likely(!err); i++) |
| err = create_chunk(heap, true); |
| |
| if (unlikely(err)) |
| delete_all_chunks(heap); |
| |
| return err; |
| } |
| |
| /** |
| * delete_heap - Delete a tiler heap |
| * |
| * This function frees any chunks allocated for a tiler heap previously |
| * initialized by @kbase_csf_tiler_heap_init and removes it from the list of |
| * heaps associated with the kbase context. The heap context structure used by |
| * the firmware is also freed. |
| * |
| * @heap: Pointer to a tiler heap to be deleted. |
| */ |
| static void delete_heap(struct kbase_csf_tiler_heap *heap) |
| { |
| struct kbase_context *const kctx = heap->kctx; |
| |
| dev_dbg(kctx->kbdev->dev, "Deleting tiler heap 0x%llX\n", heap->gpu_va); |
| |
| lockdep_assert_held(&kctx->csf.tiler_heaps.lock); |
| |
| delete_all_chunks(heap); |
| |
| /* We could optimize context destruction by not freeing leaked heap |
| * contexts but it doesn't seem worth the extra complexity. |
| */ |
| kbase_csf_heap_context_allocator_free(&kctx->csf.tiler_heaps.ctx_alloc, |
| heap->gpu_va); |
| |
| list_del(&heap->link); |
| |
| WARN_ON(heap->chunk_count); |
| KBASE_TLSTREAM_AUX_TILER_HEAP_STATS(kctx->kbdev, kctx->id, |
| heap->heap_id, 0, 0, heap->max_chunks, heap->chunk_size, 0, |
| heap->target_in_flight, 0); |
| |
| kfree(heap); |
| } |
| |
| /** |
| * find_tiler_heap - Find a tiler heap from the address of its heap context |
| * |
| * Each tiler heap managed by the kernel has an associated heap context |
| * structure used by the firmware. This function finds a tiler heap object from |
| * the GPU virtual address of its associated heap context. The heap context |
| * should have been allocated by @kbase_csf_heap_context_allocator_alloc in the |
| * same @kctx. |
| * |
| * @kctx: Pointer to the kbase context to search for a tiler heap. |
| * @heap_gpu_va: GPU virtual address of a heap context structure. |
| * |
| * Return: pointer to the tiler heap object, or NULL if not found. |
| */ |
| static struct kbase_csf_tiler_heap *find_tiler_heap( |
| struct kbase_context *const kctx, u64 const heap_gpu_va) |
| { |
| struct kbase_csf_tiler_heap *heap = NULL; |
| |
| lockdep_assert_held(&kctx->csf.tiler_heaps.lock); |
| |
| list_for_each_entry(heap, &kctx->csf.tiler_heaps.list, link) { |
| if (heap_gpu_va == heap->gpu_va) |
| return heap; |
| } |
| |
| dev_dbg(kctx->kbdev->dev, "Tiler heap 0x%llX was not found\n", |
| heap_gpu_va); |
| |
| return NULL; |
| } |
| |
| int kbase_csf_tiler_heap_context_init(struct kbase_context *const kctx) |
| { |
| int err = kbase_csf_heap_context_allocator_init( |
| &kctx->csf.tiler_heaps.ctx_alloc, kctx); |
| |
| if (unlikely(err)) |
| return err; |
| |
| INIT_LIST_HEAD(&kctx->csf.tiler_heaps.list); |
| mutex_init(&kctx->csf.tiler_heaps.lock); |
| |
| dev_dbg(kctx->kbdev->dev, "Initialized a context for tiler heaps\n"); |
| |
| return 0; |
| } |
| |
| void kbase_csf_tiler_heap_context_term(struct kbase_context *const kctx) |
| { |
| struct list_head *entry = NULL, *tmp = NULL; |
| |
| dev_dbg(kctx->kbdev->dev, "Terminating a context for tiler heaps\n"); |
| |
| mutex_lock(&kctx->csf.tiler_heaps.lock); |
| |
| list_for_each_safe(entry, tmp, &kctx->csf.tiler_heaps.list) { |
| struct kbase_csf_tiler_heap *heap = list_entry( |
| entry, struct kbase_csf_tiler_heap, link); |
| delete_heap(heap); |
| } |
| |
| mutex_unlock(&kctx->csf.tiler_heaps.lock); |
| mutex_destroy(&kctx->csf.tiler_heaps.lock); |
| |
| kbase_csf_heap_context_allocator_term(&kctx->csf.tiler_heaps.ctx_alloc); |
| } |
| |
| int kbase_csf_tiler_heap_init(struct kbase_context *const kctx, |
| u32 const chunk_size, u32 const initial_chunks, u32 const max_chunks, |
| u16 const target_in_flight, u64 *const heap_gpu_va, |
| u64 *const first_chunk_va) |
| { |
| int err = 0; |
| struct kbase_csf_tiler_heap *heap = NULL; |
| struct kbase_csf_heap_context_allocator *const ctx_alloc = |
| &kctx->csf.tiler_heaps.ctx_alloc; |
| |
| dev_dbg(kctx->kbdev->dev, |
| "Creating a tiler heap with %u chunks (limit: %u) of size %u\n", |
| initial_chunks, max_chunks, chunk_size); |
| |
| if (chunk_size == 0) |
| return -EINVAL; |
| |
| if (chunk_size & ~CHUNK_SIZE_MASK) |
| return -EINVAL; |
| |
| if (initial_chunks == 0) |
| return -EINVAL; |
| |
| if (initial_chunks > max_chunks) |
| return -EINVAL; |
| |
| if (target_in_flight == 0) |
| return -EINVAL; |
| |
| heap = kzalloc(sizeof(*heap), GFP_KERNEL); |
| if (unlikely(!heap)) { |
| dev_err(kctx->kbdev->dev, |
| "No kernel memory for a new tiler heap\n"); |
| return -ENOMEM; |
| } |
| |
| heap->kctx = kctx; |
| heap->chunk_size = chunk_size; |
| heap->max_chunks = max_chunks; |
| heap->target_in_flight = target_in_flight; |
| INIT_LIST_HEAD(&heap->chunks_list); |
| |
| heap->gpu_va = kbase_csf_heap_context_allocator_alloc(ctx_alloc); |
| |
| mutex_lock(&kctx->csf.tiler_heaps.lock); |
| |
| if (unlikely(!heap->gpu_va)) { |
| dev_err(kctx->kbdev->dev, |
| "Failed to allocate a tiler heap context\n"); |
| err = -ENOMEM; |
| } else { |
| err = create_initial_chunks(heap, initial_chunks); |
| if (unlikely(err)) { |
| kbase_csf_heap_context_allocator_free(ctx_alloc, |
| heap->gpu_va); |
| } |
| } |
| |
| if (unlikely(err)) { |
| kfree(heap); |
| } else { |
| struct kbase_csf_tiler_heap_chunk const *first_chunk = |
| list_first_entry(&heap->chunks_list, |
| struct kbase_csf_tiler_heap_chunk, link); |
| |
| kctx->csf.tiler_heaps.nr_of_heaps++; |
| heap->heap_id = kctx->csf.tiler_heaps.nr_of_heaps; |
| list_add(&heap->link, &kctx->csf.tiler_heaps.list); |
| |
| *heap_gpu_va = heap->gpu_va; |
| *first_chunk_va = first_chunk->gpu_va; |
| |
| KBASE_TLSTREAM_AUX_TILER_HEAP_STATS( |
| kctx->kbdev, kctx->id, heap->heap_id, |
| PFN_UP(heap->chunk_size * heap->max_chunks), |
| PFN_UP(heap->chunk_size * heap->chunk_count), |
| heap->max_chunks, heap->chunk_size, heap->chunk_count, |
| heap->target_in_flight, 0); |
| |
| dev_dbg(kctx->kbdev->dev, "Created tiler heap 0x%llX\n", |
| heap->gpu_va); |
| } |
| |
| mutex_unlock(&kctx->csf.tiler_heaps.lock); |
| |
| return err; |
| } |
| |
| int kbase_csf_tiler_heap_term(struct kbase_context *const kctx, |
| u64 const heap_gpu_va) |
| { |
| int err = 0; |
| struct kbase_csf_tiler_heap *heap = NULL; |
| |
| mutex_lock(&kctx->csf.tiler_heaps.lock); |
| |
| heap = find_tiler_heap(kctx, heap_gpu_va); |
| if (likely(heap)) |
| delete_heap(heap); |
| else |
| err = -EINVAL; |
| |
| mutex_unlock(&kctx->csf.tiler_heaps.lock); |
| |
| return err; |
| } |
| |
| /** |
| * alloc_new_chunk - Allocate a new chunk for the tiler heap. |
| * |
| * This function will allocate a new chunk for the chunked tiler heap depending |
| * on the settings provided by userspace when the heap was created and the |
| * heap's statistics (like number of render passes in-flight). |
| * |
| * @heap: Pointer to the tiler heap. |
| * @nr_in_flight: Number of render passes that are in-flight, must not be zero. |
| * @pending_frag_count: Number of render passes in-flight with completed vertex/tiler stage. |
| * The minimum value is zero but it must be less or equal to |
| * the total number of render passes in flight |
| * @new_chunk_ptr: Where to store the GPU virtual address & size of the new |
| * chunk allocated for the heap. |
| * |
| * Return: 0 if a new chunk was allocated otherwise an appropriate negative |
| * error code. |
| */ |
| static int alloc_new_chunk(struct kbase_csf_tiler_heap *heap, |
| u32 nr_in_flight, u32 pending_frag_count, u64 *new_chunk_ptr) |
| { |
| int err = -ENOMEM; |
| |
| lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); |
| |
| if (WARN_ON(!nr_in_flight) || |
| WARN_ON(pending_frag_count > nr_in_flight)) |
| return -EINVAL; |
| |
| if (nr_in_flight <= heap->target_in_flight) { |
| if (heap->chunk_count < heap->max_chunks) { |
| /* Not exceeded the target number of render passes yet so be |
| * generous with memory. |
| */ |
| err = create_chunk(heap, false); |
| |
| if (likely(!err)) { |
| struct kbase_csf_tiler_heap_chunk *new_chunk = |
| get_last_chunk(heap); |
| if (!WARN_ON(!new_chunk)) { |
| *new_chunk_ptr = |
| encode_chunk_ptr(heap->chunk_size, |
| new_chunk->gpu_va); |
| return 0; |
| } |
| } |
| } else if (pending_frag_count > 0) { |
| err = -EBUSY; |
| } else { |
| err = -ENOMEM; |
| } |
| } else { |
| /* Reached target number of render passes in flight. |
| * Wait for some of them to finish |
| */ |
| err = -EBUSY; |
| } |
| |
| return err; |
| } |
| |
| int kbase_csf_tiler_heap_alloc_new_chunk(struct kbase_context *kctx, |
| u64 gpu_heap_va, u32 nr_in_flight, u32 pending_frag_count, u64 *new_chunk_ptr) |
| { |
| struct kbase_csf_tiler_heap *heap; |
| int err = -EINVAL; |
| |
| mutex_lock(&kctx->csf.tiler_heaps.lock); |
| |
| heap = find_tiler_heap(kctx, gpu_heap_va); |
| |
| if (likely(heap)) { |
| err = alloc_new_chunk(heap, nr_in_flight, pending_frag_count, |
| new_chunk_ptr); |
| |
| KBASE_TLSTREAM_AUX_TILER_HEAP_STATS( |
| kctx->kbdev, kctx->id, heap->heap_id, |
| PFN_UP(heap->chunk_size * heap->max_chunks), |
| PFN_UP(heap->chunk_size * heap->chunk_count), |
| heap->max_chunks, heap->chunk_size, heap->chunk_count, |
| heap->target_in_flight, nr_in_flight); |
| } |
| |
| mutex_unlock(&kctx->csf.tiler_heaps.lock); |
| |
| return err; |
| } |