dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c

// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
 *
 * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU license.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
 *
 */

#include <tl/mali_kbase_tracepoints.h>

#include "mali_kbase_csf_tiler_heap.h"
#include "mali_kbase_csf_tiler_heap_def.h"
#include "mali_kbase_csf_heap_context_alloc.h"

/**
 * encode_chunk_ptr - Encode the address and size of a chunk as an integer.
 *
 * The size and address of the next chunk in a list are packed into a single
 * 64-bit value for storage in a chunk's header. This function returns that
 * value.
 *
 * @chunk_size: Size of a tiler heap chunk, in bytes.
 * @chunk_addr: GPU virtual address of the same tiler heap chunk.
 *
 * Return: Next chunk pointer suitable for writing into a chunk header.
 */
static u64 encode_chunk_ptr(u32 const chunk_size, u64 const chunk_addr)
{
	u64 encoded_size, encoded_addr;

	WARN_ON(chunk_size & ~CHUNK_SIZE_MASK);
	WARN_ON(chunk_addr & ~CHUNK_ADDR_MASK);

	encoded_size =
		(u64)(chunk_size >> CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT) <<
		CHUNK_HDR_NEXT_SIZE_POS;

	encoded_addr =
		(chunk_addr >> CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT) <<
		CHUNK_HDR_NEXT_ADDR_POS;

	return (encoded_size & CHUNK_HDR_NEXT_SIZE_MASK) |
		(encoded_addr & CHUNK_HDR_NEXT_ADDR_MASK);
}

/**
 * get_last_chunk - Get the last chunk of a tiler heap
 *
 * @heap:  Pointer to the tiler heap.
 *
 * Return: The address of the most recently-linked chunk, or NULL if none.
 */
static struct kbase_csf_tiler_heap_chunk *get_last_chunk(
	struct kbase_csf_tiler_heap *const heap)
{
	lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock);

	if (list_empty(&heap->chunks_list))
		return NULL;

	return list_last_entry(&heap->chunks_list,
		struct kbase_csf_tiler_heap_chunk, link);
}

/**
 * link_chunk - Link a chunk into a tiler heap
 *
 * Unless the @chunk is the first in the kernel's list of chunks belonging to
 * a given tiler heap, this function stores the size and address of the @chunk
 * in the header of the preceding chunk. This requires the GPU memory region
 * containing the header to be be mapped temporarily, which can fail.
 *
 * @heap:  Pointer to the tiler heap.
 * @chunk: Pointer to the heap chunk to be linked.
 *
 * Return: 0 if successful or a negative error code on failure.
 */
static int link_chunk(struct kbase_csf_tiler_heap *const heap,
	struct kbase_csf_tiler_heap_chunk *const chunk)
{
	struct kbase_csf_tiler_heap_chunk *const prev = get_last_chunk(heap);

	if (prev) {
		struct kbase_context *const kctx = heap->kctx;
		struct kbase_vmap_struct map;
		u64 *const prev_hdr = kbase_vmap_prot(kctx, prev->gpu_va,
			sizeof(*prev_hdr), KBASE_REG_CPU_WR, &map);

		if (unlikely(!prev_hdr)) {
			dev_err(kctx->kbdev->dev,
				"Failed to map tiler heap chunk 0x%llX\n",
				prev->gpu_va);
			return -ENOMEM;
		}

		*prev_hdr = encode_chunk_ptr(heap->chunk_size, chunk->gpu_va);
		kbase_vunmap(kctx, &map);

		dev_dbg(kctx->kbdev->dev,
			"Linked tiler heap chunks, 0x%llX -> 0x%llX\n",
			prev->gpu_va, chunk->gpu_va);
	}

	return 0;
}

/**
 * init_chunk - Initialize and link a tiler heap chunk
 *
 * Zero-initialize a new chunk's header (including its pointer to the next
 * chunk, which doesn't exist yet) and then update the previous chunk's
 * header to link the new chunk into the chunk list.
 *
 * @heap:  Pointer to the tiler heap.
 * @chunk: Pointer to the heap chunk to be initialized and linked.
 * @link_with_prev: Flag to indicate if the new chunk needs to be linked with
 *                  the previously allocated chunk.
 *
 * Return: 0 if successful or a negative error code on failure.
 */
static int init_chunk(struct kbase_csf_tiler_heap *const heap,
	struct kbase_csf_tiler_heap_chunk *const chunk, bool link_with_prev)
{
	struct kbase_vmap_struct map;
	struct u64 *chunk_hdr = NULL;
	struct kbase_context *const kctx = heap->kctx;

	if (unlikely(chunk->gpu_va & ~CHUNK_ADDR_MASK)) {
		dev_err(kctx->kbdev->dev,
			"Tiler heap chunk address is unusable\n");
		return -EINVAL;
	}

	chunk_hdr = kbase_vmap_prot(kctx,
		chunk->gpu_va, CHUNK_HDR_SIZE, KBASE_REG_CPU_WR, &map);

	if (unlikely(!chunk_hdr)) {
		dev_err(kctx->kbdev->dev,
			"Failed to map a tiler heap chunk header\n");
		return -ENOMEM;
	}

	memset(chunk_hdr, 0, CHUNK_HDR_SIZE);
	kbase_vunmap(kctx, &map);

	if (link_with_prev)
		return link_chunk(heap, chunk);
	else
		return 0;
}

/**
 * create_chunk - Create a tiler heap chunk
 *
 * This function allocates a chunk of memory for a tiler heap and adds it to
 * the end of the list of chunks associated with that heap. The size of the
 * chunk is not a parameter because it is configured per-heap not per-chunk.
 *
 * @heap: Pointer to the tiler heap for which to allocate memory.
 * @link_with_prev: Flag to indicate if the chunk to be allocated needs to be
 *                  linked with the previously allocated chunk.
 *
 * Return: 0 if successful or a negative error code on failure.
 */
static int create_chunk(struct kbase_csf_tiler_heap *const heap,
		bool link_with_prev)
{
	int err = 0;
	struct kbase_context *const kctx = heap->kctx;
	u64 nr_pages = PFN_UP(heap->chunk_size);
	u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR |
		BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE |
		BASE_MEM_COHERENT_LOCAL;
	struct kbase_csf_tiler_heap_chunk *chunk = NULL;

	flags |= base_mem_group_id_set(kctx->jit_group_id);

#if defined(CONFIG_MALI_DEBUG) || defined(CONFIG_MALI_VECTOR_DUMP)
	flags |= BASE_MEM_PROT_CPU_RD;
#endif

	lockdep_assert_held(&kctx->csf.tiler_heaps.lock);

	chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
	if (unlikely(!chunk)) {
		dev_err(kctx->kbdev->dev,
			"No kernel memory for a new tiler heap chunk\n");
		return -ENOMEM;
	}

	/* Allocate GPU memory for the new chunk. */
	INIT_LIST_HEAD(&chunk->link);
	chunk->region = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0,
		&flags, &chunk->gpu_va);

	if (unlikely(!chunk->region)) {
		dev_err(kctx->kbdev->dev,
			"Failed to allocate a tiler heap chunk\n");
		err = -ENOMEM;
	} else {
		err = init_chunk(heap, chunk, link_with_prev);
		if (unlikely(err)) {
			kbase_gpu_vm_lock(kctx);
			chunk->region->flags &= ~KBASE_REG_NO_USER_FREE;
			kbase_mem_free_region(kctx, chunk->region);
			kbase_gpu_vm_unlock(kctx);
		}
	}

	if (unlikely(err)) {
		kfree(chunk);
	} else {
		list_add_tail(&chunk->link, &heap->chunks_list);
		heap->chunk_count++;

		dev_dbg(kctx->kbdev->dev, "Created tiler heap chunk 0x%llX\n",
			chunk->gpu_va);
	}

	return err;
}

/**
 * delete_chunk - Delete a tiler heap chunk
 *
 * This function frees a tiler heap chunk previously allocated by @create_chunk
 * and removes it from the list of chunks associated with the heap.
 *
 * WARNING: The deleted chunk is not unlinked from the list of chunks used by
 *          the GPU, therefore it is only safe to use this function when
 *          deleting a heap.
 *
 * @heap:  Pointer to the tiler heap for which @chunk was allocated.
 * @chunk: Pointer to a chunk to be deleted.
 */
static void delete_chunk(struct kbase_csf_tiler_heap *const heap,
	struct kbase_csf_tiler_heap_chunk *const chunk)
{
	struct kbase_context *const kctx = heap->kctx;

	lockdep_assert_held(&kctx->csf.tiler_heaps.lock);

	kbase_gpu_vm_lock(kctx);
	chunk->region->flags &= ~KBASE_REG_NO_USER_FREE;
	kbase_mem_free_region(kctx, chunk->region);
	kbase_gpu_vm_unlock(kctx);
	list_del(&chunk->link);
	heap->chunk_count--;
	kfree(chunk);
}

/**
 * delete_all_chunks - Delete all chunks belonging to a tiler heap
 *
 * This function empties the list of chunks associated with a tiler heap by
 * freeing all chunks previously allocated by @create_chunk.
 *
 * @heap: Pointer to a tiler heap.
 */
static void delete_all_chunks(struct kbase_csf_tiler_heap *heap)
{
	struct list_head *entry = NULL, *tmp = NULL;
	struct kbase_context *const kctx = heap->kctx;

	lockdep_assert_held(&kctx->csf.tiler_heaps.lock);

	list_for_each_safe(entry, tmp, &heap->chunks_list) {
		struct kbase_csf_tiler_heap_chunk *chunk = list_entry(
			entry, struct kbase_csf_tiler_heap_chunk, link);

		delete_chunk(heap, chunk);
	}
}

/**
 * create_initial_chunks - Create the initial list of chunks for a tiler heap
 *
 * This function allocates a given number of chunks for a tiler heap and
 * adds them to the list of chunks associated with that heap.
 *
 * @heap:    Pointer to the tiler heap for which to allocate memory.
 * @nchunks: Number of chunks to create.
 *
 * Return: 0 if successful or a negative error code on failure.
 */
static int create_initial_chunks(struct kbase_csf_tiler_heap *const heap,
	u32 const nchunks)
{
	int err = 0;
	u32 i;

	for (i = 0; (i < nchunks) && likely(!err); i++)
		err = create_chunk(heap, true);

	if (unlikely(err))
		delete_all_chunks(heap);

	return err;
}

/**
 * delete_heap - Delete a tiler heap
 *
 * This function frees any chunks allocated for a tiler heap previously
 * initialized by @kbase_csf_tiler_heap_init and removes it from the list of
 * heaps associated with the kbase context. The heap context structure used by
 * the firmware is also freed.
 *
 * @heap: Pointer to a tiler heap to be deleted.
 */
static void delete_heap(struct kbase_csf_tiler_heap *heap)
{
	struct kbase_context *const kctx = heap->kctx;

	dev_dbg(kctx->kbdev->dev, "Deleting tiler heap 0x%llX\n", heap->gpu_va);

	lockdep_assert_held(&kctx->csf.tiler_heaps.lock);

	delete_all_chunks(heap);

	/* We could optimize context destruction by not freeing leaked heap
	 * contexts but it doesn't seem worth the extra complexity.
	 */
	kbase_csf_heap_context_allocator_free(&kctx->csf.tiler_heaps.ctx_alloc,
		heap->gpu_va);

	list_del(&heap->link);

	WARN_ON(heap->chunk_count);
	KBASE_TLSTREAM_AUX_TILER_HEAP_STATS(kctx->kbdev, kctx->id,
		heap->heap_id, 0, 0, heap->max_chunks, heap->chunk_size, 0,
		heap->target_in_flight, 0);

	kfree(heap);
}

/**
 * find_tiler_heap - Find a tiler heap from the address of its heap context
 *
 * Each tiler heap managed by the kernel has an associated heap context
 * structure used by the firmware. This function finds a tiler heap object from
 * the GPU virtual address of its associated heap context. The heap context
 * should have been allocated by @kbase_csf_heap_context_allocator_alloc in the
 * same @kctx.
 *
 * @kctx:        Pointer to the kbase context to search for a tiler heap.
 * @heap_gpu_va: GPU virtual address of a heap context structure.
 *
 * Return: pointer to the tiler heap object, or NULL if not found.
 */
static struct kbase_csf_tiler_heap *find_tiler_heap(
	struct kbase_context *const kctx, u64 const heap_gpu_va)
{
	struct kbase_csf_tiler_heap *heap = NULL;

	lockdep_assert_held(&kctx->csf.tiler_heaps.lock);

	list_for_each_entry(heap, &kctx->csf.tiler_heaps.list, link) {
		if (heap_gpu_va == heap->gpu_va)
			return heap;
	}

	dev_dbg(kctx->kbdev->dev, "Tiler heap 0x%llX was not found\n",
		heap_gpu_va);

	return NULL;
}

int kbase_csf_tiler_heap_context_init(struct kbase_context *const kctx)
{
	int err = kbase_csf_heap_context_allocator_init(
		&kctx->csf.tiler_heaps.ctx_alloc, kctx);

	if (unlikely(err))
		return err;

	INIT_LIST_HEAD(&kctx->csf.tiler_heaps.list);
	mutex_init(&kctx->csf.tiler_heaps.lock);

	dev_dbg(kctx->kbdev->dev, "Initialized a context for tiler heaps\n");

	return 0;
}

void kbase_csf_tiler_heap_context_term(struct kbase_context *const kctx)
{
	struct list_head *entry = NULL, *tmp = NULL;

	dev_dbg(kctx->kbdev->dev, "Terminating a context for tiler heaps\n");

	mutex_lock(&kctx->csf.tiler_heaps.lock);

	list_for_each_safe(entry, tmp, &kctx->csf.tiler_heaps.list) {
		struct kbase_csf_tiler_heap *heap = list_entry(
			entry, struct kbase_csf_tiler_heap, link);
		delete_heap(heap);
	}

	mutex_unlock(&kctx->csf.tiler_heaps.lock);
	mutex_destroy(&kctx->csf.tiler_heaps.lock);

	kbase_csf_heap_context_allocator_term(&kctx->csf.tiler_heaps.ctx_alloc);
}

int kbase_csf_tiler_heap_init(struct kbase_context *const kctx,
	u32 const chunk_size, u32 const initial_chunks, u32 const max_chunks,
	u16 const target_in_flight, u64 *const heap_gpu_va,
	u64 *const first_chunk_va)
{
	int err = 0;
	struct kbase_csf_tiler_heap *heap = NULL;
	struct kbase_csf_heap_context_allocator *const ctx_alloc =
		&kctx->csf.tiler_heaps.ctx_alloc;

	dev_dbg(kctx->kbdev->dev,
		"Creating a tiler heap with %u chunks (limit: %u) of size %u\n",
		initial_chunks, max_chunks, chunk_size);

	if (chunk_size == 0)
		return -EINVAL;

	if (chunk_size & ~CHUNK_SIZE_MASK)
		return -EINVAL;

	if (initial_chunks == 0)
		return -EINVAL;

	if (initial_chunks > max_chunks)
		return -EINVAL;

	if (target_in_flight == 0)
		return -EINVAL;

	heap = kzalloc(sizeof(*heap), GFP_KERNEL);
	if (unlikely(!heap)) {
		dev_err(kctx->kbdev->dev,
			"No kernel memory for a new tiler heap\n");
		return -ENOMEM;
	}

	heap->kctx = kctx;
	heap->chunk_size = chunk_size;
	heap->max_chunks = max_chunks;
	heap->target_in_flight = target_in_flight;
	INIT_LIST_HEAD(&heap->chunks_list);

	heap->gpu_va = kbase_csf_heap_context_allocator_alloc(ctx_alloc);

	mutex_lock(&kctx->csf.tiler_heaps.lock);

	if (unlikely(!heap->gpu_va)) {
		dev_err(kctx->kbdev->dev,
			"Failed to allocate a tiler heap context\n");
		err = -ENOMEM;
	} else {
		err = create_initial_chunks(heap, initial_chunks);
		if (unlikely(err)) {
			kbase_csf_heap_context_allocator_free(ctx_alloc,
				heap->gpu_va);
		}
	}

	if (unlikely(err)) {
		kfree(heap);
	} else {
		struct kbase_csf_tiler_heap_chunk const *first_chunk =
			list_first_entry(&heap->chunks_list,
				struct kbase_csf_tiler_heap_chunk, link);

		kctx->csf.tiler_heaps.nr_of_heaps++;
		heap->heap_id = kctx->csf.tiler_heaps.nr_of_heaps;
		list_add(&heap->link, &kctx->csf.tiler_heaps.list);

		*heap_gpu_va = heap->gpu_va;
		*first_chunk_va = first_chunk->gpu_va;

		KBASE_TLSTREAM_AUX_TILER_HEAP_STATS(
			kctx->kbdev, kctx->id, heap->heap_id,
			PFN_UP(heap->chunk_size * heap->max_chunks),
			PFN_UP(heap->chunk_size * heap->chunk_count),
			heap->max_chunks, heap->chunk_size, heap->chunk_count,
			heap->target_in_flight, 0);

		dev_dbg(kctx->kbdev->dev, "Created tiler heap 0x%llX\n",
			heap->gpu_va);
	}

	mutex_unlock(&kctx->csf.tiler_heaps.lock);

	return err;
}

int kbase_csf_tiler_heap_term(struct kbase_context *const kctx,
	u64 const heap_gpu_va)
{
	int err = 0;
	struct kbase_csf_tiler_heap *heap = NULL;

	mutex_lock(&kctx->csf.tiler_heaps.lock);

	heap = find_tiler_heap(kctx, heap_gpu_va);
	if (likely(heap))
		delete_heap(heap);
	else
		err = -EINVAL;

	mutex_unlock(&kctx->csf.tiler_heaps.lock);

	return err;
}

/**
 * alloc_new_chunk - Allocate a new chunk for the tiler heap.
 *
 * This function will allocate a new chunk for the chunked tiler heap depending
 * on the settings provided by userspace when the heap was created and the
 * heap's statistics (like number of render passes in-flight).
 *
 * @heap:               Pointer to the tiler heap.
 * @nr_in_flight:       Number of render passes that are in-flight, must not be zero.
 * @pending_frag_count: Number of render passes in-flight with completed vertex/tiler stage.
 *                      The minimum value is zero but it must be less or equal to
 *                      the total number of render passes in flight
 * @new_chunk_ptr:      Where to store the GPU virtual address & size of the new
 *                      chunk allocated for the heap.
 *
 * Return: 0 if a new chunk was allocated otherwise an appropriate negative
 *         error code.
 */
static int alloc_new_chunk(struct kbase_csf_tiler_heap *heap,
		u32 nr_in_flight, u32 pending_frag_count, u64 *new_chunk_ptr)
{
	int err = -ENOMEM;

	lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock);

	if (WARN_ON(!nr_in_flight) ||
		WARN_ON(pending_frag_count > nr_in_flight))
		return -EINVAL;

	if (nr_in_flight <= heap->target_in_flight) {
		if (heap->chunk_count < heap->max_chunks) {
			/* Not exceeded the target number of render passes yet so be
			 * generous with memory.
			 */
			err = create_chunk(heap, false);

			if (likely(!err)) {
				struct kbase_csf_tiler_heap_chunk *new_chunk =
								get_last_chunk(heap);
				if (!WARN_ON(!new_chunk)) {
					*new_chunk_ptr =
						encode_chunk_ptr(heap->chunk_size,
								 new_chunk->gpu_va);
					return 0;
				}
			}
		} else if (pending_frag_count > 0) {
			err = -EBUSY;
		} else {
			err = -ENOMEM;
		}
	} else {
		/* Reached target number of render passes in flight.
		 * Wait for some of them to finish
		 */
		err = -EBUSY;
	}

	return err;
}

int kbase_csf_tiler_heap_alloc_new_chunk(struct kbase_context *kctx,
	u64 gpu_heap_va, u32 nr_in_flight, u32 pending_frag_count, u64 *new_chunk_ptr)
{
	struct kbase_csf_tiler_heap *heap;
	int err = -EINVAL;

	mutex_lock(&kctx->csf.tiler_heaps.lock);

	heap = find_tiler_heap(kctx, gpu_heap_va);

	if (likely(heap)) {
		err = alloc_new_chunk(heap, nr_in_flight, pending_frag_count,
			new_chunk_ptr);

		KBASE_TLSTREAM_AUX_TILER_HEAP_STATS(
			kctx->kbdev, kctx->id, heap->heap_id,
			PFN_UP(heap->chunk_size * heap->max_chunks),
			PFN_UP(heap->chunk_size * heap->chunk_count),
			heap->max_chunks, heap->chunk_size, heap->chunk_count,
			heap->target_in_flight, nr_in_flight);
	}

	mutex_unlock(&kctx->csf.tiler_heaps.lock);

	return err;
}