bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_gpu_narrow.c - manifest_repos/mali-driver - Git at Google

 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2021-2022 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
  * Foundation, and any use by you of this program is subject to the terms
  * of such GNU license.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, you can access it online at
  * http://www.gnu.org/licenses/gpl-2.0.html.
  *
  */

 #include "mali_kbase_hwcnt_gpu.h"
 #include "mali_kbase_hwcnt_gpu_narrow.h"

 #include <linux/bug.h>
 #include <linux/err.h>
 #include <linux/slab.h>

 int kbase_hwcnt_gpu_metadata_narrow_create(
 	const struct kbase_hwcnt_metadata_narrow **dst_md_narrow,
 	const struct kbase_hwcnt_metadata *src_md)
 {
 	struct kbase_hwcnt_description desc;
 	struct kbase_hwcnt_group_description group;
 	struct kbase_hwcnt_block_description
 		blks[KBASE_HWCNT_V5_BLOCK_TYPE_COUNT];
 	size_t prfcnt_values_per_block;
 	size_t blk;
 	int err;
 	struct kbase_hwcnt_metadata_narrow *metadata_narrow;

 	if (!dst_md_narrow || !src_md || !src_md->grp_metadata ||
 	    !src_md->grp_metadata[0].blk_metadata)
 		return -EINVAL;

 	/* Only support 1 group count and KBASE_HWCNT_V5_BLOCK_TYPE_COUNT block
 	 * count in the metadata.
 	 */
 	if ((kbase_hwcnt_metadata_group_count(src_md) != 1) ||
 	    (kbase_hwcnt_metadata_block_count(src_md, 0) !=
 	     KBASE_HWCNT_V5_BLOCK_TYPE_COUNT))
 		return -EINVAL;

 	/* Get the values count in the first block. */
 	prfcnt_values_per_block =
 		kbase_hwcnt_metadata_block_values_count(src_md, 0, 0);

 	/* check all blocks should have same values count. */
 	for (blk = 1; blk < KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; blk++) {
 		size_t val_cnt =
 			kbase_hwcnt_metadata_block_values_count(src_md, 0, blk);
 		if (val_cnt != prfcnt_values_per_block)
 			return -EINVAL;
 	}

 	/* Only support 64 and 128 entries per block. */
 	if ((prfcnt_values_per_block != 64) && (prfcnt_values_per_block != 128))
 		return -EINVAL;

 	metadata_narrow = kmalloc(sizeof(*metadata_narrow), GFP_KERNEL);
 	if (!metadata_narrow)
 		return -ENOMEM;

 	/* Narrow to 64 entries per block to keep API backward compatibility. */
 	prfcnt_values_per_block = 64;

 	for (blk = 0; blk < KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; blk++) {
 		size_t blk_hdr_cnt = kbase_hwcnt_metadata_block_headers_count(
 			src_md, 0, blk);
 		blks[blk] = (struct kbase_hwcnt_block_description){
 			.type = kbase_hwcnt_metadata_block_type(src_md, 0, blk),
 			.inst_cnt = kbase_hwcnt_metadata_block_instance_count(
 				src_md, 0, blk),
 			.hdr_cnt = blk_hdr_cnt,
 			.ctr_cnt = prfcnt_values_per_block - blk_hdr_cnt,
 		};
 	}

 	group = (struct kbase_hwcnt_group_description){
 		.type = kbase_hwcnt_metadata_group_type(src_md, 0),
 		.blk_cnt = KBASE_HWCNT_V5_BLOCK_TYPE_COUNT,
 		.blks = blks,
 	};

 	desc = (struct kbase_hwcnt_description){
 		.grp_cnt = kbase_hwcnt_metadata_group_count(src_md),
 		.avail_mask = src_md->avail_mask,
 		.clk_cnt = src_md->clk_cnt,
 		.grps = &group,
 	};

 	err = kbase_hwcnt_metadata_create(&desc, &metadata_narrow->metadata);
 	if (!err) {
 		/* Narrow down the buffer size to half as the narrowed metadata
 		 * only supports 32-bit but the created metadata uses 64-bit for
 		 * block entry.
 		 */
 		metadata_narrow->dump_buf_bytes =
 			metadata_narrow->metadata->dump_buf_bytes >> 1;
 		*dst_md_narrow = metadata_narrow;
 	} else {
 		kfree(metadata_narrow);
 	}

 	return err;
 }

 void kbase_hwcnt_gpu_metadata_narrow_destroy(
 	const struct kbase_hwcnt_metadata_narrow *md_narrow)
 {
 	if (!md_narrow)
 		return;

 	kbase_hwcnt_metadata_destroy(md_narrow->metadata);
 	kfree(md_narrow);
 }

 int kbase_hwcnt_dump_buffer_narrow_alloc(
 	const struct kbase_hwcnt_metadata_narrow *md_narrow,
 	struct kbase_hwcnt_dump_buffer_narrow *dump_buf)
 {
 	size_t dump_buf_bytes;
 	size_t clk_cnt_buf_bytes;
 	u8 *buf;

 	if (!md_narrow || !dump_buf)
 		return -EINVAL;

 	dump_buf_bytes = md_narrow->dump_buf_bytes;
 	clk_cnt_buf_bytes =
 		sizeof(*dump_buf->clk_cnt_buf) * md_narrow->metadata->clk_cnt;

 	/* Make a single allocation for both dump_buf and clk_cnt_buf. */
 	buf = kmalloc(dump_buf_bytes + clk_cnt_buf_bytes, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;

 	*dump_buf = (struct kbase_hwcnt_dump_buffer_narrow){
 		.md_narrow = md_narrow,
 		.dump_buf = (u32 *)buf,
 		.clk_cnt_buf = (u64 *)(buf + dump_buf_bytes),
 	};

 	return 0;
 }

 void kbase_hwcnt_dump_buffer_narrow_free(
 	struct kbase_hwcnt_dump_buffer_narrow *dump_buf_narrow)
 {
 	if (!dump_buf_narrow)
 		return;

 	kfree(dump_buf_narrow->dump_buf);
 	*dump_buf_narrow = (struct kbase_hwcnt_dump_buffer_narrow){ NULL };
 }

 int kbase_hwcnt_dump_buffer_narrow_array_alloc(
 	const struct kbase_hwcnt_metadata_narrow *md_narrow, size_t n,
 	struct kbase_hwcnt_dump_buffer_narrow_array *dump_bufs)
 {
 	struct kbase_hwcnt_dump_buffer_narrow *buffers;
 	size_t buf_idx;
 	unsigned int order;
 	unsigned long addr;
 	size_t dump_buf_bytes;
 	size_t clk_cnt_buf_bytes;
 	size_t total_dump_buf_size;

 	if (!md_narrow || !dump_bufs)
 		return -EINVAL;

 	dump_buf_bytes = md_narrow->dump_buf_bytes;
 	clk_cnt_buf_bytes = sizeof(*dump_bufs->bufs->clk_cnt_buf) *
 			    md_narrow->metadata->clk_cnt;

 	/* Allocate memory for the dump buffer struct array */
 	buffers = kmalloc_array(n, sizeof(*buffers), GFP_KERNEL);
 	if (!buffers)
 		return -ENOMEM;

 	/* Allocate pages for the actual dump buffers, as they tend to be fairly
 	 * large.
 	 */
 	order = get_order((dump_buf_bytes + clk_cnt_buf_bytes) * n);
 	addr = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);

 	if (!addr) {
 		kfree(buffers);
 		return -ENOMEM;
 	}

 	*dump_bufs = (struct kbase_hwcnt_dump_buffer_narrow_array){
 		.page_addr = addr,
 		.page_order = order,
 		.buf_cnt = n,
 		.bufs = buffers,
 	};

 	total_dump_buf_size = dump_buf_bytes * n;
 	/* Set the buffer of each dump buf */
 	for (buf_idx = 0; buf_idx < n; buf_idx++) {
 		const size_t dump_buf_offset = dump_buf_bytes * buf_idx;
 		const size_t clk_cnt_buf_offset =
 			total_dump_buf_size + (clk_cnt_buf_bytes * buf_idx);

 		buffers[buf_idx] = (struct kbase_hwcnt_dump_buffer_narrow){
 			.md_narrow = md_narrow,
 			.dump_buf = (u32 *)(addr + dump_buf_offset),
 			.clk_cnt_buf = (u64 *)(addr + clk_cnt_buf_offset),
 		};
 	}

 	return 0;
 }

 void kbase_hwcnt_dump_buffer_narrow_array_free(
 	struct kbase_hwcnt_dump_buffer_narrow_array *dump_bufs)
 {
 	if (!dump_bufs)
 		return;

 	kfree(dump_bufs->bufs);
 	free_pages(dump_bufs->page_addr, dump_bufs->page_order);
 	memset(dump_bufs, 0, sizeof(*dump_bufs));
 }

 void kbase_hwcnt_dump_buffer_block_copy_strict_narrow(u32 *dst_blk,
 						      const u64 *src_blk,
 						      const u64 *blk_em,
 						      size_t val_cnt)
 {
 	size_t val;

 	for (val = 0; val < val_cnt; val++) {
 		bool val_enabled =
 			kbase_hwcnt_enable_map_block_value_enabled(blk_em, val);
 		u32 src_val =
 			(src_blk[val] > U32_MAX) ? U32_MAX : (u32)src_blk[val];

 		dst_blk[val] = val_enabled ? src_val : 0;
 	}
 }

 void kbase_hwcnt_dump_buffer_copy_strict_narrow(
 	struct kbase_hwcnt_dump_buffer_narrow *dst_narrow,
 	const struct kbase_hwcnt_dump_buffer *src,
 	const struct kbase_hwcnt_enable_map *dst_enable_map)
 {
 	const struct kbase_hwcnt_metadata_narrow *metadata_narrow;
 	size_t grp;
 	size_t clk;

 	if (WARN_ON(!dst_narrow) || WARN_ON(!src) || WARN_ON(!dst_enable_map) ||
 	    WARN_ON(dst_narrow->md_narrow->metadata == src->metadata) ||
 	    WARN_ON(dst_narrow->md_narrow->metadata->grp_cnt !=
 		    src->metadata->grp_cnt) ||
 	    WARN_ON(src->metadata->grp_cnt != 1) ||
 	    WARN_ON(dst_narrow->md_narrow->metadata->grp_metadata[0].blk_cnt !=
 		    src->metadata->grp_metadata[0].blk_cnt) ||
 	    WARN_ON(dst_narrow->md_narrow->metadata->grp_metadata[0].blk_cnt !=
 		    KBASE_HWCNT_V5_BLOCK_TYPE_COUNT) ||
 	    WARN_ON(dst_narrow->md_narrow->metadata->grp_metadata[0]
 			    .blk_metadata[0]
 			    .ctr_cnt >
 		    src->metadata->grp_metadata[0].blk_metadata[0].ctr_cnt))
 		return;

 	/* Don't use src metadata since src buffer is bigger than dst buffer. */
 	metadata_narrow = dst_narrow->md_narrow;

 	for (grp = 0;
 	     grp < kbase_hwcnt_metadata_narrow_group_count(metadata_narrow);
 	     grp++) {
 		size_t blk;
 		size_t blk_cnt = kbase_hwcnt_metadata_narrow_block_count(
 			metadata_narrow, grp);

 		for (blk = 0; blk < blk_cnt; blk++) {
 			size_t blk_inst;
 			size_t blk_inst_cnt =
 				kbase_hwcnt_metadata_narrow_block_instance_count(
 					metadata_narrow, grp, blk);

 			for (blk_inst = 0; blk_inst < blk_inst_cnt;
 			     blk_inst++) {
 				/* The narrowed down buffer is only 32-bit. */
 				u32 *dst_blk =
 					kbase_hwcnt_dump_buffer_narrow_block_instance(
 						dst_narrow, grp, blk, blk_inst);
 				const u64 *src_blk =
 					kbase_hwcnt_dump_buffer_block_instance(
 						src, grp, blk, blk_inst);
 				const u64 *blk_em =
 					kbase_hwcnt_enable_map_block_instance(
 						dst_enable_map, grp, blk,
 						blk_inst);
 				size_t val_cnt =
 					kbase_hwcnt_metadata_narrow_block_values_count(
 						metadata_narrow, grp, blk);
 				/* Align upwards to include padding bytes */
 				val_cnt = KBASE_HWCNT_ALIGN_UPWARDS(
 					val_cnt,
 					(KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT /
 					 KBASE_HWCNT_VALUE_BYTES));

 				kbase_hwcnt_dump_buffer_block_copy_strict_narrow(
 					dst_blk, src_blk, blk_em, val_cnt);
 			}
 		}
 	}

 	for (clk = 0; clk < metadata_narrow->metadata->clk_cnt; clk++) {
 		bool clk_enabled = kbase_hwcnt_clk_enable_map_enabled(
 			dst_enable_map->clk_enable_map, clk);

 		dst_narrow->clk_cnt_buf[clk] =
 			clk_enabled ? src->clk_cnt_buf[clk] : 0;
 	}
 }
	// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
	/*
	*
	* (C) COPYRIGHT 2021-2022 ARM Limited. All rights reserved.
	*
	* This program is free software and is provided to you under the terms of the
	* GNU General Public License version 2 as published by the Free Software
	* Foundation, and any use by you of this program is subject to the terms
	* of such GNU license.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public License
	* along with this program; if not, you can access it online at
	* http://www.gnu.org/licenses/gpl-2.0.html.
	*
	*/

	#include "mali_kbase_hwcnt_gpu.h"
	#include "mali_kbase_hwcnt_gpu_narrow.h"

	#include <linux/bug.h>
	#include <linux/err.h>
	#include <linux/slab.h>

	int kbase_hwcnt_gpu_metadata_narrow_create(
	const struct kbase_hwcnt_metadata_narrow **dst_md_narrow,
	const struct kbase_hwcnt_metadata *src_md)
	{
	struct kbase_hwcnt_description desc;
	struct kbase_hwcnt_group_description group;
	struct kbase_hwcnt_block_description
	blks[KBASE_HWCNT_V5_BLOCK_TYPE_COUNT];
	size_t prfcnt_values_per_block;
	size_t blk;
	int err;
	struct kbase_hwcnt_metadata_narrow *metadata_narrow;

	if (!dst_md_narrow \|\| !src_md \|\| !src_md->grp_metadata \|\|
	!src_md->grp_metadata[0].blk_metadata)
	return -EINVAL;

	/* Only support 1 group count and KBASE_HWCNT_V5_BLOCK_TYPE_COUNT block
	* count in the metadata.
	*/
	if ((kbase_hwcnt_metadata_group_count(src_md) != 1) \|\|
	(kbase_hwcnt_metadata_block_count(src_md, 0) !=
	KBASE_HWCNT_V5_BLOCK_TYPE_COUNT))
	return -EINVAL;

	/* Get the values count in the first block. */
	prfcnt_values_per_block =
	kbase_hwcnt_metadata_block_values_count(src_md, 0, 0);

	/* check all blocks should have same values count. */
	for (blk = 1; blk < KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; blk++) {
	size_t val_cnt =
	kbase_hwcnt_metadata_block_values_count(src_md, 0, blk);
	if (val_cnt != prfcnt_values_per_block)
	return -EINVAL;
	}

	/* Only support 64 and 128 entries per block. */
	if ((prfcnt_values_per_block != 64) && (prfcnt_values_per_block != 128))
	return -EINVAL;

	metadata_narrow = kmalloc(sizeof(*metadata_narrow), GFP_KERNEL);
	if (!metadata_narrow)
	return -ENOMEM;

	/* Narrow to 64 entries per block to keep API backward compatibility. */
	prfcnt_values_per_block = 64;

	for (blk = 0; blk < KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; blk++) {
	size_t blk_hdr_cnt = kbase_hwcnt_metadata_block_headers_count(
	src_md, 0, blk);
	blks[blk] = (struct kbase_hwcnt_block_description){
	.type = kbase_hwcnt_metadata_block_type(src_md, 0, blk),
	.inst_cnt = kbase_hwcnt_metadata_block_instance_count(
	src_md, 0, blk),
	.hdr_cnt = blk_hdr_cnt,
	.ctr_cnt = prfcnt_values_per_block - blk_hdr_cnt,
	};
	}

	group = (struct kbase_hwcnt_group_description){
	.type = kbase_hwcnt_metadata_group_type(src_md, 0),
	.blk_cnt = KBASE_HWCNT_V5_BLOCK_TYPE_COUNT,
	.blks = blks,
	};

	desc = (struct kbase_hwcnt_description){
	.grp_cnt = kbase_hwcnt_metadata_group_count(src_md),
	.avail_mask = src_md->avail_mask,
	.clk_cnt = src_md->clk_cnt,
	.grps = &group,
	};

	err = kbase_hwcnt_metadata_create(&desc, &metadata_narrow->metadata);
	if (!err) {
	/* Narrow down the buffer size to half as the narrowed metadata
	* only supports 32-bit but the created metadata uses 64-bit for
	* block entry.
	*/
	metadata_narrow->dump_buf_bytes =
	metadata_narrow->metadata->dump_buf_bytes >> 1;
	*dst_md_narrow = metadata_narrow;
	} else {
	kfree(metadata_narrow);
	}

	return err;
	}

	void kbase_hwcnt_gpu_metadata_narrow_destroy(
	const struct kbase_hwcnt_metadata_narrow *md_narrow)
	{
	if (!md_narrow)
	return;

	kbase_hwcnt_metadata_destroy(md_narrow->metadata);
	kfree(md_narrow);
	}

	int kbase_hwcnt_dump_buffer_narrow_alloc(
	const struct kbase_hwcnt_metadata_narrow *md_narrow,
	struct kbase_hwcnt_dump_buffer_narrow *dump_buf)
	{
	size_t dump_buf_bytes;
	size_t clk_cnt_buf_bytes;
	u8 *buf;

	if (!md_narrow \|\| !dump_buf)
	return -EINVAL;

	dump_buf_bytes = md_narrow->dump_buf_bytes;
	clk_cnt_buf_bytes =
	sizeof(dump_buf->clk_cnt_buf) md_narrow->metadata->clk_cnt;

	/* Make a single allocation for both dump_buf and clk_cnt_buf. */
	buf = kmalloc(dump_buf_bytes + clk_cnt_buf_bytes, GFP_KERNEL);
	if (!buf)
	return -ENOMEM;

	*dump_buf = (struct kbase_hwcnt_dump_buffer_narrow){
	.md_narrow = md_narrow,
	.dump_buf = (u32 *)buf,
	.clk_cnt_buf = (u64 *)(buf + dump_buf_bytes),
	};

	return 0;
	}

	void kbase_hwcnt_dump_buffer_narrow_free(
	struct kbase_hwcnt_dump_buffer_narrow *dump_buf_narrow)
	{
	if (!dump_buf_narrow)
	return;

	kfree(dump_buf_narrow->dump_buf);
	*dump_buf_narrow = (struct kbase_hwcnt_dump_buffer_narrow){ NULL };
	}

	int kbase_hwcnt_dump_buffer_narrow_array_alloc(
	const struct kbase_hwcnt_metadata_narrow *md_narrow, size_t n,
	struct kbase_hwcnt_dump_buffer_narrow_array *dump_bufs)
	{
	struct kbase_hwcnt_dump_buffer_narrow *buffers;
	size_t buf_idx;
	unsigned int order;
	unsigned long addr;
	size_t dump_buf_bytes;
	size_t clk_cnt_buf_bytes;
	size_t total_dump_buf_size;

	if (!md_narrow \|\| !dump_bufs)
	return -EINVAL;

	dump_buf_bytes = md_narrow->dump_buf_bytes;
	clk_cnt_buf_bytes = sizeof(dump_bufs->bufs->clk_cnt_buf)
	md_narrow->metadata->clk_cnt;

	/* Allocate memory for the dump buffer struct array */
	buffers = kmalloc_array(n, sizeof(*buffers), GFP_KERNEL);
	if (!buffers)
	return -ENOMEM;

	/* Allocate pages for the actual dump buffers, as they tend to be fairly
	* large.
	*/
	order = get_order((dump_buf_bytes + clk_cnt_buf_bytes) * n);
	addr = __get_free_pages(GFP_KERNEL \| __GFP_ZERO, order);

	if (!addr) {
	kfree(buffers);
	return -ENOMEM;
	}

	*dump_bufs = (struct kbase_hwcnt_dump_buffer_narrow_array){
	.page_addr = addr,
	.page_order = order,
	.buf_cnt = n,
	.bufs = buffers,
	};

	total_dump_buf_size = dump_buf_bytes * n;
	/* Set the buffer of each dump buf */
	for (buf_idx = 0; buf_idx < n; buf_idx++) {
	const size_t dump_buf_offset = dump_buf_bytes * buf_idx;
	const size_t clk_cnt_buf_offset =
	total_dump_buf_size + (clk_cnt_buf_bytes * buf_idx);

	buffers[buf_idx] = (struct kbase_hwcnt_dump_buffer_narrow){
	.md_narrow = md_narrow,
	.dump_buf = (u32 *)(addr + dump_buf_offset),
	.clk_cnt_buf = (u64 *)(addr + clk_cnt_buf_offset),
	};
	}

	return 0;
	}

	void kbase_hwcnt_dump_buffer_narrow_array_free(
	struct kbase_hwcnt_dump_buffer_narrow_array *dump_bufs)
	{
	if (!dump_bufs)
	return;

	kfree(dump_bufs->bufs);
	free_pages(dump_bufs->page_addr, dump_bufs->page_order);
	memset(dump_bufs, 0, sizeof(*dump_bufs));
	}

	void kbase_hwcnt_dump_buffer_block_copy_strict_narrow(u32 *dst_blk,
	const u64 *src_blk,
	const u64 *blk_em,
	size_t val_cnt)
	{
	size_t val;

	for (val = 0; val < val_cnt; val++) {
	bool val_enabled =
	kbase_hwcnt_enable_map_block_value_enabled(blk_em, val);
	u32 src_val =
	(src_blk[val] > U32_MAX) ? U32_MAX : (u32)src_blk[val];

	dst_blk[val] = val_enabled ? src_val : 0;
	}
	}

	void kbase_hwcnt_dump_buffer_copy_strict_narrow(
	struct kbase_hwcnt_dump_buffer_narrow *dst_narrow,
	const struct kbase_hwcnt_dump_buffer *src,
	const struct kbase_hwcnt_enable_map *dst_enable_map)
	{
	const struct kbase_hwcnt_metadata_narrow *metadata_narrow;
	size_t grp;
	size_t clk;

	if (WARN_ON(!dst_narrow) \|\| WARN_ON(!src) \|\| WARN_ON(!dst_enable_map) \|\|
	WARN_ON(dst_narrow->md_narrow->metadata == src->metadata) \|\|
	WARN_ON(dst_narrow->md_narrow->metadata->grp_cnt !=
	src->metadata->grp_cnt) \|\|
	WARN_ON(src->metadata->grp_cnt != 1) \|\|
	WARN_ON(dst_narrow->md_narrow->metadata->grp_metadata[0].blk_cnt !=
	src->metadata->grp_metadata[0].blk_cnt) \|\|
	WARN_ON(dst_narrow->md_narrow->metadata->grp_metadata[0].blk_cnt !=
	KBASE_HWCNT_V5_BLOCK_TYPE_COUNT) \|\|
	WARN_ON(dst_narrow->md_narrow->metadata->grp_metadata[0]
	.blk_metadata[0]
	.ctr_cnt >
	src->metadata->grp_metadata[0].blk_metadata[0].ctr_cnt))
	return;

	/* Don't use src metadata since src buffer is bigger than dst buffer. */
	metadata_narrow = dst_narrow->md_narrow;

	for (grp = 0;
	grp < kbase_hwcnt_metadata_narrow_group_count(metadata_narrow);
	grp++) {
	size_t blk;
	size_t blk_cnt = kbase_hwcnt_metadata_narrow_block_count(
	metadata_narrow, grp);

	for (blk = 0; blk < blk_cnt; blk++) {
	size_t blk_inst;
	size_t blk_inst_cnt =
	kbase_hwcnt_metadata_narrow_block_instance_count(
	metadata_narrow, grp, blk);

	for (blk_inst = 0; blk_inst < blk_inst_cnt;
	blk_inst++) {
	/* The narrowed down buffer is only 32-bit. */
	u32 *dst_blk =
	kbase_hwcnt_dump_buffer_narrow_block_instance(
	dst_narrow, grp, blk, blk_inst);
	const u64 *src_blk =
	kbase_hwcnt_dump_buffer_block_instance(
	src, grp, blk, blk_inst);
	const u64 *blk_em =
	kbase_hwcnt_enable_map_block_instance(
	dst_enable_map, grp, blk,
	blk_inst);
	size_t val_cnt =
	kbase_hwcnt_metadata_narrow_block_values_count(
	metadata_narrow, grp, blk);
	/* Align upwards to include padding bytes */
	val_cnt = KBASE_HWCNT_ALIGN_UPWARDS(
	val_cnt,
	(KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT /
	KBASE_HWCNT_VALUE_BYTES));

	kbase_hwcnt_dump_buffer_block_copy_strict_narrow(
	dst_blk, src_blk, blk_em, val_cnt);
	}
	}
	}

	for (clk = 0; clk < metadata_narrow->metadata->clk_cnt; clk++) {
	bool clk_enabled = kbase_hwcnt_clk_enable_map_enabled(
	dst_enable_map->clk_enable_map, clk);

	dst_narrow->clk_cnt_buf[clk] =
	clk_enabled ? src->clk_cnt_buf[clk] : 0;
	}
	}