dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if_fw.c

// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
 *
 * (C) COPYRIGHT 2021 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU license.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
 *
 */

/*
 * CSF GPU HWC backend firmware interface APIs.
 */

#include <mali_kbase.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <device/mali_kbase_device.h>
#include "mali_kbase_hwcnt_gpu.h"
#include "mali_kbase_hwcnt_types.h"
#include <uapi/gpu/arm/midgard/csf/mali_gpu_csf_registers.h>

#include "csf/mali_kbase_csf_firmware.h"
#include "mali_kbase_hwcnt_backend_csf_if_fw.h"
#include "mali_kbase_hwaccess_time.h"
#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"

#include <linux/log2.h>
#include "mali_kbase_ccswe.h"


/** The number of nanoseconds in a second. */
#define NSECS_IN_SEC 1000000000ull /* ns */

/* Ring buffer virtual address start at 4GB  */
#define KBASE_HWC_CSF_RING_BUFFER_VA_START (1ull << 32)

/**
 * struct kbase_hwcnt_backend_csf_if_fw_ring_buf - ring buffer for CSF interface
 *                                                 used to save the manual and
 *                                                 auto HWC samples from
 *                                                 firmware.
 * @gpu_dump_base: Starting GPU base address of the ring buffer.
 * @cpu_dump_base: Starting CPU address for the mapping.
 * @buf_count:     Buffer count in the ring buffer, MUST be power of 2.
 * @as_nr:         Address space number for the memory mapping.
 * @phys:          Physical memory allocation used by the mapping.
 * @num_pages:     Size of the mapping, in memory pages.
 */
struct kbase_hwcnt_backend_csf_if_fw_ring_buf {
	u64 gpu_dump_base;
	void *cpu_dump_base;
	size_t buf_count;
	u32 as_nr;
	struct tagged_addr *phys;
	size_t num_pages;
};

/**
 * struct kbase_hwcnt_backend_csf_if_fw_ctx - Firmware context for the CSF
 *                                            interface, used to communicate
 *                                            with firmware.
 * @kbdev:              KBase device.
 * @buf_bytes:	        The size in bytes for each buffer in the ring buffer.
 * @clk_cnt:            The number of clock domains in the system.
 *                      The maximum is 64.
 * @clk_enable_map:     Bitmask of enabled clocks
 * @rate_listener:      Clock rate listener callback state.
 * @ccswe_shader_cores: Shader cores cycle count software estimator.
 */
struct kbase_hwcnt_backend_csf_if_fw_ctx {
	struct kbase_device *kbdev;
	size_t buf_bytes;
	u8 clk_cnt;
	u64 clk_enable_map;
	struct kbase_clk_rate_listener rate_listener;
	struct kbase_ccswe ccswe_shader_cores;
};

static void kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx)
{
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx;
	struct kbase_device *kbdev;

	WARN_ON(!ctx);

	fw_ctx = (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;
	kbdev = fw_ctx->kbdev;

	kbase_csf_scheduler_spin_lock_assert_held(kbdev);
}

static void
kbasep_hwcnt_backend_csf_if_fw_lock(struct kbase_hwcnt_backend_csf_if_ctx *ctx,
				    unsigned long *flags)
{
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx;
	struct kbase_device *kbdev;

	WARN_ON(!ctx);

	fw_ctx = (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;
	kbdev = fw_ctx->kbdev;

	kbase_csf_scheduler_spin_lock(kbdev, flags);
}

static void kbasep_hwcnt_backend_csf_if_fw_unlock(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx, unsigned long flags)
{
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx;
	struct kbase_device *kbdev;

	WARN_ON(!ctx);

	fw_ctx = (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;
	kbdev = fw_ctx->kbdev;

	kbase_csf_scheduler_spin_lock_assert_held(kbdev);
	kbase_csf_scheduler_spin_unlock(kbdev, flags);
}

/**
 * kbasep_hwcnt_backend_csf_if_fw_on_freq_change() - On freq change callback
 *
 * @rate_listener:    Callback state
 * @clk_index:        Clock index
 * @clk_rate_hz:      Clock frequency(hz)
 */
static void kbasep_hwcnt_backend_csf_if_fw_on_freq_change(
	struct kbase_clk_rate_listener *rate_listener, u32 clk_index,
	u32 clk_rate_hz)
{
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
		container_of(rate_listener,
			     struct kbase_hwcnt_backend_csf_if_fw_ctx,
			     rate_listener);
	u64 timestamp_ns;

	if (clk_index != KBASE_CLOCK_DOMAIN_SHADER_CORES)
		return;

	timestamp_ns = ktime_get_raw_ns();
	kbase_ccswe_freq_change(&fw_ctx->ccswe_shader_cores, timestamp_ns,
				clk_rate_hz);
}

/**
 * kbasep_hwcnt_backend_csf_if_fw_cc_enable() - Enable cycle count tracking
 *
 * @fw_ctx:         Non-NULL pointer to CSF firmware interface context.
 * @clk_enable_map: Non-NULL pointer to enable map specifying enabled counters.
 */
static void kbasep_hwcnt_backend_csf_if_fw_cc_enable(
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx, u64 clk_enable_map)
{
	struct kbase_device *kbdev = fw_ctx->kbdev;

	if (kbase_hwcnt_clk_enable_map_enabled(
		    clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) {
		/* software estimation for non-top clock domains */
		struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm;
		const struct kbase_clk_data *clk_data =
			rtm->clks[KBASE_CLOCK_DOMAIN_SHADER_CORES];
		u32 cur_freq;
		unsigned long flags;
		u64 timestamp_ns;

		timestamp_ns = ktime_get_raw_ns();

		spin_lock_irqsave(&rtm->lock, flags);

		cur_freq = (u32)clk_data->clock_val;
		kbase_ccswe_reset(&fw_ctx->ccswe_shader_cores);
		kbase_ccswe_freq_change(&fw_ctx->ccswe_shader_cores,
					timestamp_ns, cur_freq);

		kbase_clk_rate_trace_manager_subscribe_no_lock(
			rtm, &fw_ctx->rate_listener);

		spin_unlock_irqrestore(&rtm->lock, flags);
	}

	fw_ctx->clk_enable_map = clk_enable_map;
}

/**
 * kbasep_hwcnt_backend_csf_if_fw_cc_disable() - Disable cycle count tracking
 *
 * @fw_ctx:     Non-NULL pointer to CSF firmware interface context.
 */
static void kbasep_hwcnt_backend_csf_if_fw_cc_disable(
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx)
{
	struct kbase_device *kbdev = fw_ctx->kbdev;
	struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm;
	u64 clk_enable_map = fw_ctx->clk_enable_map;

	if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map,
					       KBASE_CLOCK_DOMAIN_SHADER_CORES))
		kbase_clk_rate_trace_manager_unsubscribe(
			rtm, &fw_ctx->rate_listener);
}

static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx,
	struct kbase_hwcnt_backend_csf_if_prfcnt_info *prfcnt_info)
{
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx;
	struct kbase_device *kbdev;
	u32 prfcnt_size;
	u32 prfcnt_hw_size = 0;
	u32 prfcnt_fw_size = 0;
	u32 prfcnt_block_size = KBASE_HWCNT_V5_DEFAULT_VALUES_PER_BLOCK *
				KBASE_HWCNT_VALUE_BYTES;

	WARN_ON(!ctx);
	WARN_ON(!prfcnt_info);

	fw_ctx = (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;
	kbdev = fw_ctx->kbdev;
	prfcnt_size = kbdev->csf.global_iface.prfcnt_size;
	prfcnt_hw_size = (prfcnt_size & 0xFF) << 8;
	prfcnt_fw_size = (prfcnt_size >> 16) << 8;
	fw_ctx->buf_bytes = prfcnt_hw_size + prfcnt_fw_size;


	prfcnt_info->dump_bytes = fw_ctx->buf_bytes;
	prfcnt_info->prfcnt_block_size = prfcnt_block_size;
	prfcnt_info->l2_count = kbdev->gpu_props.props.l2_props.num_l2_slices;
	prfcnt_info->core_mask =
		kbdev->gpu_props.props.coherency_info.group[0].core_mask;

	prfcnt_info->clk_cnt = fw_ctx->clk_cnt;
	prfcnt_info->clearing_samples = true;

	/* Block size must be multiple of counter size. */
	WARN_ON((prfcnt_info->prfcnt_block_size % KBASE_HWCNT_VALUE_BYTES) !=
		0);
	/* Total size must be multiple of block size. */
	WARN_ON((prfcnt_info->dump_bytes % prfcnt_info->prfcnt_block_size) !=
		0);
}

static int kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 buf_count,
	void **cpu_dump_base,
	struct kbase_hwcnt_backend_csf_if_ring_buf **out_ring_buf)
{
	struct kbase_device *kbdev;
	struct tagged_addr *phys;
	struct page **page_list;
	void *cpu_addr;
	int ret;
	int i;
	size_t num_pages;
	u64 flags;
	struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf;

	pgprot_t cpu_map_prot = PAGE_KERNEL;
	u64 gpu_va_base = KBASE_HWC_CSF_RING_BUFFER_VA_START;

	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
		(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;

	WARN_ON(!ctx);
	WARN_ON(!cpu_dump_base);
	WARN_ON(!out_ring_buf);

	kbdev = fw_ctx->kbdev;

	/* The buffer count must be power of 2 */
	if (!is_power_of_2(buf_count))
		return -EINVAL;

	/* alignment failure */
	if (gpu_va_base & (2048 - 1))
		return -EINVAL;

	fw_ring_buf = kzalloc(sizeof(*fw_ring_buf), GFP_KERNEL);
	if (!fw_ring_buf)
		return -ENOMEM;

	num_pages = PFN_UP(fw_ctx->buf_bytes * buf_count);
	phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL);
	if (!phys)
		goto phys_alloc_error;

	page_list = kmalloc_array(num_pages, sizeof(*page_list), GFP_KERNEL);
	if (!page_list)
		goto page_list_alloc_error;

	/* Get physical page for the buffer */
	ret = kbase_mem_pool_alloc_pages(
		&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages,
		phys, false);
	if (ret != num_pages)
		goto phys_mem_pool_alloc_error;

	/* Get the CPU virtual address */
	for (i = 0; i < num_pages; i++)
		page_list[i] = as_page(phys[i]);

	cpu_addr = vmap(page_list, num_pages, VM_MAP, cpu_map_prot);
	if (!cpu_addr)
		goto vmap_error;

	flags = KBASE_REG_GPU_WR | KBASE_REG_GPU_NX |
		KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE);

	/* Update MMU table */
	ret = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu,
				     gpu_va_base >> PAGE_SHIFT, phys, num_pages,
				     flags, MCU_AS_NR, KBASE_MEM_GROUP_CSF_FW);
	if (ret)
		goto mmu_insert_failed;

	kfree(page_list);

	fw_ring_buf->gpu_dump_base = gpu_va_base;
	fw_ring_buf->cpu_dump_base = cpu_addr;
	fw_ring_buf->phys = phys;
	fw_ring_buf->num_pages = num_pages;
	fw_ring_buf->buf_count = buf_count;
	fw_ring_buf->as_nr = MCU_AS_NR;

	*cpu_dump_base = fw_ring_buf->cpu_dump_base;
	*out_ring_buf =
		(struct kbase_hwcnt_backend_csf_if_ring_buf *)fw_ring_buf;


	return 0;

mmu_insert_failed:
	vunmap(cpu_addr);
vmap_error:
	kbase_mem_pool_free_pages(
		&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages,
		phys, false, false);
phys_mem_pool_alloc_error:
	kfree(page_list);
page_list_alloc_error:
	kfree(phys);
phys_alloc_error:
	kfree(fw_ring_buf);
	return -ENOMEM;
}

static void kbasep_hwcnt_backend_csf_if_fw_ring_buf_sync(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx,
	struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf,
	u32 buf_index_first, u32 buf_index_last, bool for_cpu)
{
	struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf =
		(struct kbase_hwcnt_backend_csf_if_fw_ring_buf *)ring_buf;
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
		(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;
	size_t i;
	size_t pg_first;
	size_t pg_last;
	u64 start_address;
	u64 stop_address;
	u32 ring_buf_index_first;
	u32 ring_buf_index_last;

	WARN_ON(!ctx);
	WARN_ON(!ring_buf);

	/* The index arguments for this function form an inclusive, exclusive
	 * range.
	 * However, when masking back to the available buffers we will make this
	 * inclusive at both ends so full flushes are not 0 -> 0.
	 */
	ring_buf_index_first = buf_index_first & (fw_ring_buf->buf_count - 1);
	ring_buf_index_last =
		(buf_index_last - 1) & (fw_ring_buf->buf_count - 1);

	/* The start address is the offset of the first buffer. */
	start_address = fw_ctx->buf_bytes * ring_buf_index_first;
	pg_first = start_address >> PAGE_SHIFT;

	/* The stop address is the last byte in the final buffer. */
	stop_address = (fw_ctx->buf_bytes * (ring_buf_index_last + 1)) - 1;
	pg_last = stop_address >> PAGE_SHIFT;

	/* Check whether the buffer range wraps. */
	if (start_address > stop_address) {
		/* sync the first part to the end of ring buffer. */
		for (i = pg_first; i < fw_ring_buf->num_pages; i++) {
			struct page *pg = as_page(fw_ring_buf->phys[i]);

			if (for_cpu) {
				kbase_sync_single_for_cpu(fw_ctx->kbdev,
							  kbase_dma_addr(pg),
							  PAGE_SIZE,
							  DMA_BIDIRECTIONAL);
			} else {
				kbase_sync_single_for_device(fw_ctx->kbdev,
							     kbase_dma_addr(pg),
							     PAGE_SIZE,
							     DMA_BIDIRECTIONAL);
			}
		}

		/* second part starts from page 0. */
		pg_first = 0;
	}

	for (i = pg_first; i <= pg_last; i++) {
		struct page *pg = as_page(fw_ring_buf->phys[i]);

		if (for_cpu) {
			kbase_sync_single_for_cpu(fw_ctx->kbdev,
						  kbase_dma_addr(pg), PAGE_SIZE,
						  DMA_BIDIRECTIONAL);
		} else {
			kbase_sync_single_for_device(fw_ctx->kbdev,
						     kbase_dma_addr(pg),
						     PAGE_SIZE,
						     DMA_BIDIRECTIONAL);
		}
	}
}

static u64 kbasep_hwcnt_backend_csf_if_fw_timestamp_ns(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx)
{
	CSTD_UNUSED(ctx);
	return ktime_get_raw_ns();
}

static void kbasep_hwcnt_backend_csf_if_fw_ring_buf_free(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx,
	struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf)
{
	struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf =
		(struct kbase_hwcnt_backend_csf_if_fw_ring_buf *)ring_buf;
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
		(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;

	if (!fw_ring_buf)
		return;

	if (fw_ring_buf->phys) {
		u64 gpu_va_base = KBASE_HWC_CSF_RING_BUFFER_VA_START;

		WARN_ON(kbase_mmu_teardown_pages(
			fw_ctx->kbdev, &fw_ctx->kbdev->csf.mcu_mmu,
			gpu_va_base >> PAGE_SHIFT, fw_ring_buf->num_pages,
			MCU_AS_NR));

		vunmap(fw_ring_buf->cpu_dump_base);

		kbase_mem_pool_free_pages(
			&fw_ctx->kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW],
			fw_ring_buf->num_pages, fw_ring_buf->phys, false,
			false);

		kfree(fw_ring_buf->phys);

		kfree(fw_ring_buf);
	}
}

static void kbasep_hwcnt_backend_csf_if_fw_dump_enable(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx,
	struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf,
	struct kbase_hwcnt_backend_csf_if_enable *enable)
{
	u32 prfcnt_config;
	struct kbase_device *kbdev;
	struct kbase_csf_global_iface *global_iface;
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
		(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;
	struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf =
		(struct kbase_hwcnt_backend_csf_if_fw_ring_buf *)ring_buf;

	WARN_ON(!ctx);
	WARN_ON(!ring_buf);
	WARN_ON(!enable);
	kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx);

	kbdev = fw_ctx->kbdev;
	global_iface = &kbdev->csf.global_iface;

	/* Configure */
	prfcnt_config = fw_ring_buf->buf_count;
	prfcnt_config |= enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT;

	/* Configure the ring buffer base address */
	kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_JASID,
					fw_ring_buf->as_nr);
	kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_BASE_LO,
					fw_ring_buf->gpu_dump_base & U32_MAX);
	kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_BASE_HI,
					fw_ring_buf->gpu_dump_base >> 32);

	/* Set extract position to 0 */
	kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_EXTRACT, 0);

	/* Configure the enable bitmap */
	kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_CSF_EN,
					enable->fe_bm);
	kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_SHADER_EN,
					enable->shader_bm);
	kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_MMU_L2_EN,
					enable->mmu_l2_bm);
	kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_TILER_EN,
					enable->tiler_bm);

	/* Configure the HWC set and buffer size */
	kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_CONFIG,
					prfcnt_config);

	kbdev->csf.hwcnt.enable_pending = true;

	/* Unmask the interrupts */
	kbase_csf_firmware_global_input_mask(
		global_iface, GLB_ACK_IRQ_MASK,
		GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK,
		GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK);
	kbase_csf_firmware_global_input_mask(
		global_iface, GLB_ACK_IRQ_MASK,
		GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK,
		GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK);
	kbase_csf_firmware_global_input_mask(
		global_iface, GLB_ACK_IRQ_MASK,
		GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK,
		GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK);
	kbase_csf_firmware_global_input_mask(
		global_iface, GLB_ACK_IRQ_MASK,
		GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK,
		GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK);

	/* Enable the HWC */
	kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ,
					     (1 << GLB_REQ_PRFCNT_ENABLE_SHIFT),
					     GLB_REQ_PRFCNT_ENABLE_MASK);
	kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);

	prfcnt_config = kbase_csf_firmware_global_input_read(global_iface,
							     GLB_PRFCNT_CONFIG);

	kbasep_hwcnt_backend_csf_if_fw_cc_enable(fw_ctx,
						 enable->clk_enable_map);
}

static void kbasep_hwcnt_backend_csf_if_fw_dump_disable(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx)
{
	struct kbase_device *kbdev;
	struct kbase_csf_global_iface *global_iface;
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
		(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;

	WARN_ON(!ctx);
	kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx);

	kbdev = fw_ctx->kbdev;
	global_iface = &kbdev->csf.global_iface;

	/* Disable the HWC */
	kbdev->csf.hwcnt.enable_pending = true;
	kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, 0,
					     GLB_REQ_PRFCNT_ENABLE_MASK);
	kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);

	/* mask the interrupts */
	kbase_csf_firmware_global_input_mask(
		global_iface, GLB_ACK_IRQ_MASK, 0,
		GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK);
	kbase_csf_firmware_global_input_mask(
		global_iface, GLB_ACK_IRQ_MASK, 0,
		GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK);
	kbase_csf_firmware_global_input_mask(
		global_iface, GLB_ACK_IRQ_MASK, 0,
		GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK);

	/* In case we have a previous request in flight when the disable
	 * happens.
	 */
	kbdev->csf.hwcnt.request_pending = false;

	kbasep_hwcnt_backend_csf_if_fw_cc_disable(fw_ctx);
}

static void kbasep_hwcnt_backend_csf_if_fw_dump_request(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx)
{
	u32 glb_req;
	struct kbase_device *kbdev;
	struct kbase_csf_global_iface *global_iface;
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
		(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;

	WARN_ON(!ctx);
	kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx);

	kbdev = fw_ctx->kbdev;
	global_iface = &kbdev->csf.global_iface;

	/* Trigger dumping */
	kbdev->csf.hwcnt.request_pending = true;
	glb_req = kbase_csf_firmware_global_input_read(global_iface, GLB_REQ);
	glb_req ^= GLB_REQ_PRFCNT_SAMPLE_MASK;
	kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, glb_req,
					     GLB_REQ_PRFCNT_SAMPLE_MASK);
	kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
}

static void kbasep_hwcnt_backend_csf_if_fw_get_indexes(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 *extract_index,
	u32 *insert_index)
{
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
		(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;

	WARN_ON(!ctx);
	WARN_ON(!extract_index);
	WARN_ON(!insert_index);
	kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx);

	*extract_index = kbase_csf_firmware_global_input_read(
		&fw_ctx->kbdev->csf.global_iface, GLB_PRFCNT_EXTRACT);
	*insert_index = kbase_csf_firmware_global_output(
		&fw_ctx->kbdev->csf.global_iface, GLB_PRFCNT_INSERT);
}

static void kbasep_hwcnt_backend_csf_if_fw_set_extract_index(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 extract_idx)
{
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
		(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;

	WARN_ON(!ctx);
	kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx);

	/* Set the raw extract index to release the buffer back to the ring
	 * buffer.
	 */
	kbase_csf_firmware_global_input(&fw_ctx->kbdev->csf.global_iface,
					GLB_PRFCNT_EXTRACT, extract_idx);
}

static void kbasep_hwcnt_backend_csf_if_fw_get_gpu_cycle_count(
	struct kbase_hwcnt_backend_csf_if_ctx *ctx, u64 *cycle_counts,
	u64 clk_enable_map)
{
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
		(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;
	u8 clk;
	u64 timestamp_ns = ktime_get_raw_ns();

	WARN_ON(!ctx);
	WARN_ON(!cycle_counts);
	kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx);

	for (clk = 0; clk < fw_ctx->clk_cnt; clk++) {
		if (!(clk_enable_map & (1ull << clk)))
			continue;

		if (clk == KBASE_CLOCK_DOMAIN_TOP) {
			/* Read cycle count for top clock domain. */
			kbase_backend_get_gpu_time_norequest(
				fw_ctx->kbdev, &cycle_counts[clk], NULL, NULL);
		} else {
			/* Estimate cycle count for non-top clock domain. */
			cycle_counts[clk] = kbase_ccswe_cycle_at(
				&fw_ctx->ccswe_shader_cores, timestamp_ns);
		}
	}
}

/**
 * kbasep_hwcnt_backedn_csf_if_fw_cts_destroy() - Destroy a CSF FW interface context.
 *
 * @fw_ctx: Pointer to context to destroy.
 */
static void kbasep_hwcnt_backend_csf_if_fw_ctx_destroy(
	struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx)
{
	if (!fw_ctx)
		return;

	kfree(fw_ctx);
}

/**
 * kbasep_hwcnt_backend_csf_if_fw_ctx_create() - Create a CSF Firmware context.
 *
 * @kbdev:   Non_NULL pointer to kbase device.
 * @out_ctx: Non-NULL pointer to where info is stored on success.
 * Return: 0 on success, else error code.
 */
static int kbasep_hwcnt_backend_csf_if_fw_ctx_create(
	struct kbase_device *kbdev,
	struct kbase_hwcnt_backend_csf_if_fw_ctx **out_ctx)
{
	u8 clk;
	int errcode = -ENOMEM;
	struct kbase_hwcnt_backend_csf_if_fw_ctx *ctx = NULL;

	WARN_ON(!kbdev);
	WARN_ON(!out_ctx);

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		goto error;

	ctx->kbdev = kbdev;

	/* Determine the number of available clock domains. */
	for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) {
		if (kbdev->pm.clk_rtm.clks[clk] == NULL)
			break;
	}
	ctx->clk_cnt = clk;

	ctx->clk_enable_map = 0;
	kbase_ccswe_init(&ctx->ccswe_shader_cores);
	ctx->rate_listener.notify =
		kbasep_hwcnt_backend_csf_if_fw_on_freq_change;

	*out_ctx = ctx;

	return 0;
error:
	kbasep_hwcnt_backend_csf_if_fw_ctx_destroy(ctx);
	return errcode;
}

void kbase_hwcnt_backend_csf_if_fw_destroy(
	struct kbase_hwcnt_backend_csf_if *if_fw)
{
	if (!if_fw)
		return;

	kbasep_hwcnt_backend_csf_if_fw_ctx_destroy(
		(struct kbase_hwcnt_backend_csf_if_fw_ctx *)if_fw->ctx);
	memset(if_fw, 0, sizeof(*if_fw));
}

int kbase_hwcnt_backend_csf_if_fw_create(
	struct kbase_device *kbdev, struct kbase_hwcnt_backend_csf_if *if_fw)
{
	int errcode;
	struct kbase_hwcnt_backend_csf_if_fw_ctx *ctx = NULL;

	if (!kbdev || !if_fw)
		return -EINVAL;

	errcode = kbasep_hwcnt_backend_csf_if_fw_ctx_create(kbdev, &ctx);
	if (errcode)
		return errcode;

	if_fw->ctx = (struct kbase_hwcnt_backend_csf_if_ctx *)ctx;
	if_fw->assert_lock_held =
		kbasep_hwcnt_backend_csf_if_fw_assert_lock_held;
	if_fw->lock = kbasep_hwcnt_backend_csf_if_fw_lock;
	if_fw->unlock = kbasep_hwcnt_backend_csf_if_fw_unlock;
	if_fw->get_prfcnt_info = kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info;
	if_fw->ring_buf_alloc = kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc;
	if_fw->ring_buf_sync = kbasep_hwcnt_backend_csf_if_fw_ring_buf_sync;
	if_fw->ring_buf_free = kbasep_hwcnt_backend_csf_if_fw_ring_buf_free;
	if_fw->timestamp_ns = kbasep_hwcnt_backend_csf_if_fw_timestamp_ns;
	if_fw->dump_enable = kbasep_hwcnt_backend_csf_if_fw_dump_enable;
	if_fw->dump_disable = kbasep_hwcnt_backend_csf_if_fw_dump_disable;
	if_fw->dump_request = kbasep_hwcnt_backend_csf_if_fw_dump_request;
	if_fw->get_gpu_cycle_count =
		kbasep_hwcnt_backend_csf_if_fw_get_gpu_cycle_count;
	if_fw->get_indexes = kbasep_hwcnt_backend_csf_if_fw_get_indexes;
	if_fw->set_extract_index =
		kbasep_hwcnt_backend_csf_if_fw_set_extract_index;

	return 0;
}