drivers/misc/mediatek/mtee/tz_cma.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2021 MediaTek Inc.
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/cma.h>
#include <linux/delay.h>
#include <linux/mm.h>
#include <linux/kthread.h>
#include <linux/platform_device.h>
#include <linux/dma-mapping.h>
#include <linux/sched.h>
#include <linux/sched/clock.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/of_reserved_mem.h>
#include <linux/mmzone.h>
#include <linux/gfp.h>
#include <linux/workqueue.h>
#include <linux/semaphore.h>
#include "tz_cma.h"
#include <linux/vmalloc.h>
#include <linux/version.h>
#include <linux/ctype.h>

//Feature option
#define MTEE_CMA_UHD_BUFFER_CHECK_SIZE 0x900000
#define MTEE_CMA_PARALLEL_TASK_USER_NICE -2
#define MTEE_CMA_PARALLEL_TASK_NUM 4
#define MTEE_CMA_MEM_BLK_NUM_MAX 64
#define MTEE_CMA_DELAY_FREE_CHECK_MS 5000
#define MTEE_CMA_DELAY_FREE_POLLING_MS (tz_cma_dynamic_mode ? 5 : 500)
#define MTEE_CMA_ALLOCAE_TOTAL_TIMEOUT_MS 3000
#define MTEE_CMA_ALLOCAE_TIMEOUT_MS 1000
#define MTEE_CMA_COMPLETION_TIMEOUT_MS 60000
#define MTEE_GET_SYS_MS() (sched_clock()/1000000)
#define MTEE_CMA_SLOT_SIZE_MB 4
#define MTEE_CMA_DEFAULT_SIZE (192*SZ_1M)//FHD buffer
#define MTEE_CMA_UHD_10BIT_SIZE(_num) ((_num)*15*SZ_1M)
#define MTEE_CMA_UHD_FB_NUM (14)
#define MTEE_CMA_UHD_FB_SIZE MTEE_CMA_UHD_10BIT_SIZE(12)
#define MTEE_CMA_CACHE_FOR_UHD_MEM MTEE_CMA_UHD_10BIT_SIZE(9)
#define MTEE_CMA_CACHE_FOR_UHD_PAGE_CNT (MTEE_CMA_CACHE_FOR_UHD_MEM >> PAGE_SHIFT)
#define MTEE_CMA_ALIGN_UP_SIZE(x, _align) ((((x)+_align-1)/_align)*_align)

enum CMA_REGION_STATUS {
	CMA_REGION_FREED = 0,
	CMA_REGION_ALLOCATED,
	CMA_REGION_CACHED,
	CMA_REGION_PROCESS,
};

struct tz_mem_unit {
	uint32_t pfn;
	uint32_t page_cnt;
	uint8_t allocated;
	#if defined(MTEE_CMA_CACHE_POLICY)
	uint64_t cached_time;
	#endif
};

struct parall_cma_alloc {
	struct completion task_start;
	struct completion task_finish;
};

enum {
	TZPT_SVP_MEMORY_CARVEOUT,
	TZPT_SVP_MEMORY_RESERVED_MEMORY,
	TZPT_SVP_MEMORY_CMA,
};

static struct cma *tz_cma;
static uint64_t tz_cma_base;
static size_t tz_cma_secure_size;
static size_t tz_cma_alloc_pfn_start;
static size_t tz_cma_alloc_pfn_end;
static bool tz_cma_allocated;
static struct mutex tz_cma_api_lock;
static struct mutex tz_cma_allocate_lock;
static struct platform_device *tz_cma_dev;
#if defined(MTEE_CMA_CACHE_POLICY)
static struct delayed_work dleay_free_work;
static unsigned long tz_last_free_called_ms;
static int tz_cma_delay_free_ms;
static bool tz_cma_prealloc_buffer;
static unsigned long tz_cma_prealloc_buffer_time;
static struct completion pre_alloc_task_start;
static size_t tz_cma_cache_level;
static bool tz_cma_prealloc_uhd_buffer;
static KREE_SECUREMEM_HANDLE tz_cma_first4k_buffer_handle;
static size_t tz_cma_map_region_size;
#endif

static struct tz_mem_unit *tz_mem_blocks;
static size_t tz_mem_slot_size;

#if defined(MTEE_CMA_ENABLE_PARALLEL_ALLOC)
static struct mutex tz_cma_slot_lock;
static struct parall_cma_alloc parallel_tasks[MTEE_CMA_PARALLEL_TASK_NUM];
#endif
static uint8_t tz_svp_memory_mode = TZPT_SVP_MEMORY_CARVEOUT;
#if defined(MTEE_CMA_DYNAMIC_MODE)
static bool tz_cma_dynamic_mode;
#endif
#if IS_ENABLED(CONFIG_MTEE_CMA_DEBUG)
static uint32_t tz_svp_debug_max_mem_mb;
#if defined(MTEE_CMA_CACHE_POLICY)
static unsigned long tz_svp_uhd_preallocate_ms;
#endif
#if defined(MTEE_CMA_ENABLE_PARALLEL_ALLOC)
static size_t tz_cma_alloc_debug_first_pfn;
#endif
#endif

static void  mtee_cma_update_cache_level(uint64_t size);

#if defined(MTEE_CMA_CACHE_POLICY)
static unsigned long tz_cm_shrink(int nr_to_scan, bool force_free);

static bool _mtee_cma_in_range(struct tz_mem_unit *slot, size_t pfn_start, size_t pfn_end)
{
	if (slot->pfn + slot->page_cnt <= pfn_start || slot->pfn >= pfn_end)
		return false;

	return true;
}

static void _mtee_cma_check_free(struct work_struct *work)
{
	unsigned long nr_total;
	bool check_free = false;

	if (MTEE_GET_SYS_MS() - tz_last_free_called_ms >= tz_cma_delay_free_ms)
		check_free = true;
	if (tz_cma_dynamic_mode)
		check_free = true;
	mutex_lock(&tz_cma_allocate_lock);
	mutex_lock(&tz_cma_slot_lock);
	if (!tz_cma_allocated && check_free) {
		nr_total = tz_cm_shrink(tz_cma_secure_size >> PAGE_SHIFT, tz_cma_dynamic_mode);
		pr_info("%s() free 0x%08x bytes\n", __func__, nr_total << PAGE_SHIFT);
	}
	mutex_unlock(&tz_cma_slot_lock);
	mutex_unlock(&tz_cma_allocate_lock);

	if (check_free == false)
		schedule_delayed_work(&dleay_free_work,
			msecs_to_jiffies(MTEE_CMA_DELAY_FREE_POLLING_MS));

}
#endif

static void _mtee_flush_pages(uint64_t pfn, uint32_t cnt)
{
	struct page *page = pfn_to_page(pfn);
	dma_addr_t dma_addr;

	dma_addr = dma_map_page(&tz_cma_dev->dev, page, 0, cnt << PAGE_SHIFT, DMA_TO_DEVICE);
	dma_sync_single_for_device(&tz_cma_dev->dev, dma_addr, cnt << PAGE_SHIFT, DMA_TO_DEVICE);
	dma_unmap_page(&tz_cma_dev->dev, dma_addr, cnt << PAGE_SHIFT, DMA_TO_DEVICE);
}

static bool _mtee_alloc_contig_range(struct tz_mem_unit *slot)
{
	int retries = 0;
	int ret = 0;
	unsigned long ms = MTEE_GET_SYS_MS();
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 110)
#else
	struct acr_info info;

	memset(&info, 0x0, sizeof(struct acr_info));
#endif
	for (retries = 0; retries < 10; retries++) {
		ret = alloc_contig_range(slot->pfn,
			slot->pfn + slot->page_cnt,
			MIGRATE_CMA, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 110)
			);
#else
			, &info);
#endif
		if (ret == 0) {
			_mtee_flush_pages(slot->pfn, slot->page_cnt);
			return true;
		}

		if (MTEE_GET_SYS_MS() - ms >= MTEE_CMA_ALLOCAE_TIMEOUT_MS)
			break;

		if (ret == -EBUSY) {
			usleep_range(1*1000, 2*1000);
			retries = 0;
		}

	}
	pr_info("%s (pa, size) =(0x%08x, 0x%08x) fail, diff = %d, ret = %d\n",
		__func__, (int)(slot->pfn << PAGE_SHIFT), (int)(slot->page_cnt << PAGE_SHIFT),
		(int)(MTEE_GET_SYS_MS() - ms), ret);


	return false;
}

#if defined(MTEE_CMA_ENABLE_PARALLEL_ALLOC)
static int _mtee_cma_parallel_alloc_task(void *data)
{
	struct parall_cma_alloc *task = (struct parall_cma_alloc *)data;
	int  i = 0;
	unsigned long t1;
	struct tz_mem_unit *slot;
	int slot_num;
	int total_bytes;
	size_t pfn_start;
	size_t pfn_end;
	unsigned long ms;
	bool ret = true;

	for (;;) {
		if (wait_for_completion_timeout(&task->task_start,
			msecs_to_jiffies(MTEE_CMA_COMPLETION_TIMEOUT_MS)) == 0)
			continue;
		reinit_completion(&task->task_start);
		slot_num = 0;
		total_bytes = 0;
		t1 = MTEE_GET_SYS_MS();
		pfn_start = tz_cma_alloc_pfn_start;
		pfn_end = tz_cma_alloc_pfn_end;
		ms = MTEE_GET_SYS_MS();
		do {
			mutex_lock(&tz_cma_slot_lock);
			slot = NULL;

			for (i = 0; i < tz_mem_slot_size; i++) {
				if (_mtee_cma_in_range(&tz_mem_blocks[i],
					pfn_start, pfn_end) == false)
					continue;

				if (tz_mem_blocks[i].allocated == CMA_REGION_CACHED &&
					tz_cma_prealloc_buffer == false)
					tz_mem_blocks[i].allocated = CMA_REGION_ALLOCATED;

				if (tz_mem_blocks[i].allocated == CMA_REGION_FREED) {
					slot = &tz_mem_blocks[i];
					//mark processed
					slot->allocated = CMA_REGION_PROCESS;
					#if IS_ENABLED(CONFIG_MTEE_CMA_DEBUG)
					if (tz_mem_blocks[i].pfn <= tz_cma_alloc_debug_first_pfn)
						tz_cma_alloc_debug_first_pfn = tz_mem_blocks[i].pfn;
					#endif
					break;
				}
			}
			mutex_unlock(&tz_cma_slot_lock);
			//all done, finish
			if (!slot)
				break;
			ret = _mtee_alloc_contig_range(slot);
			mutex_lock(&tz_cma_slot_lock);
			if (ret == true) {
				if (tz_cma_prealloc_buffer) {
					slot->cached_time = MTEE_GET_SYS_MS();
					slot->allocated = CMA_REGION_CACHED;
				} else
					slot->allocated = CMA_REGION_ALLOCATED;
				slot_num++;
				total_bytes += (slot->page_cnt << PAGE_SHIFT);
			} else
				slot->allocated = CMA_REGION_FREED;
			mutex_unlock(&tz_cma_slot_lock);
			if (ret == false)
				break;
			if (MTEE_GET_SYS_MS() - ms >= MTEE_CMA_ALLOCAE_TOTAL_TIMEOUT_MS) {
				pr_info("%s: timeout\n", __func__);
				break;
			}
		} while (1);
		complete(&task->task_finish);
	}
	return 0;
}
#endif

static bool _mtee_cma_free_impl_by_range(uint64_t phy, uint64_t size, bool cache)
{
	int i = 0;
	size_t pfn_start = PFN_DOWN(phy);
	size_t pfn_end = PFN_DOWN((phy + size));

	mutex_lock(&tz_cma_slot_lock);
	for (i = 0; i < tz_mem_slot_size; i++) {

		if (_mtee_cma_in_range(tz_mem_blocks + i, pfn_start, pfn_end) == false)
			continue;

		if (tz_mem_blocks[i].allocated == CMA_REGION_ALLOCATED ||
			tz_mem_blocks[i].allocated == CMA_REGION_CACHED) {
			#if defined(MTEE_CMA_CACHE_POLICY)
			if (cache) {
				//mark to freeable
				tz_mem_blocks[i].allocated = CMA_REGION_CACHED;
				tz_mem_blocks[i].cached_time = MTEE_GET_SYS_MS();
				continue;
			}
			#endif

			free_contig_range(tz_mem_blocks[i].pfn, tz_mem_blocks[i].page_cnt);
			tz_mem_blocks[i].allocated = CMA_REGION_FREED;
		}
	}
	mutex_unlock(&tz_cma_slot_lock);
	return true;
}

static bool  _mtee_cma_allocate_impl_by_range(uint64_t phy, uint64_t size, bool alloc_from_cache)
{
	bool ret = true;
	int i = 0;
	size_t pfn_start = PFN_DOWN(phy);
	size_t pfn_end = PFN_DOWN((phy + size));
	unsigned long ms = MTEE_GET_SYS_MS();
	unsigned long check_timeout_ms = ms;
#if defined(MTEE_CMA_ENABLE_PARALLEL_ALLOC)
	bool check_cache_result = 0;

	do {
		if (alloc_from_cache) {
			mutex_lock(&tz_cma_slot_lock);
			check_cache_result = false;
			for (i = 0; i < tz_mem_slot_size; i++) {
				if (_mtee_cma_in_range(&tz_mem_blocks[i], pfn_start, pfn_end) == false)
					continue;

				if (tz_mem_blocks[i].allocated == CMA_REGION_CACHED)
					check_cache_result = true;
				else {
					check_cache_result = false;
					break;
				}
			}
			if (check_cache_result) {
				pr_info("%s: get from pre-allocated memory\n", __func__);
				for (i = 0; i < tz_mem_slot_size; i++) {
					if (_mtee_cma_in_range(&tz_mem_blocks[i], pfn_start, pfn_end) == false)
						continue;

					if (tz_mem_blocks[i].allocated == CMA_REGION_CACHED)
						tz_mem_blocks[i].allocated = CMA_REGION_ALLOCATED;

				}
				mutex_unlock(&tz_cma_slot_lock);
				return true;
			}
			mutex_unlock(&tz_cma_slot_lock);
		}
		//lock fail, check in next loop and wait mutex released by task
		if (mutex_trylock(&tz_cma_allocate_lock) == 0) {
			usleep_range(5*1000, 10*1000);
			if (MTEE_GET_SYS_MS() - check_timeout_ms >= MTEE_CMA_ALLOCAE_TOTAL_TIMEOUT_MS) {
				pr_info("%s: timeout\n", __func__);
				return false;
			}
		} else
			break;
	} while (true);

	ms = MTEE_GET_SYS_MS();
	tz_cma_alloc_pfn_start = pfn_start;
	tz_cma_alloc_pfn_end = pfn_end;
	#if IS_ENABLED(CONFIG_MTEE_CMA_DEBUG)
	tz_cma_alloc_debug_first_pfn = pfn_end;
	#endif
	for (i = 0; i < MTEE_CMA_PARALLEL_TASK_NUM; i++)
		complete(&parallel_tasks[i].task_start);

	for (i = 0; i < MTEE_CMA_PARALLEL_TASK_NUM; i++) {
		wait_for_completion(&parallel_tasks[i].task_finish);
		reinit_completion(&parallel_tasks[i].task_finish);
	}
	#if IS_ENABLED(CONFIG_MTEE_CMA_DEBUG)
	if (tz_cma_alloc_debug_first_pfn != pfn_end)
		phy = (tz_cma_alloc_debug_first_pfn << PAGE_SHIFT);
	#endif
#else
	for (i = 0; i < tz_mem_slot_size; i++) {
		if (_mtee_cma_in_range(&tz_mem_blocks[i], pfn_start, pfn_end) == false)
			continue;

		if (tz_mem_blocks[i].allocated == CMA_REGION_CACHED)
			tz_mem_blocks[i].allocated = CMA_REGION_ALLOCATED;
		else if (tz_mem_blocks[i].allocated == CMA_REGION_FREED) {
			if (_mtee_alloc_contig_range(&tz_mem_blocks[i]) == true) {
				if (tz_cma_prealloc_buffer) {
					tz_mem_blocks[i].cached_time = MTEE_GET_SYS_MS();
					tz_mem_blocks[i].allocated = CMA_REGION_CACHED;
				} else
					tz_mem_blocks[i].allocated = CMA_REGION_ALLOCATED;
			} else {
				pr_info("%s alloc_contig_range(%x~%x) fail\n",
					__func__,
					(int)(tz_mem_blocks[i].pfn << PAGE_SHIFT),
					(int)((tz_mem_blocks[i].pfn + tz_mem_blocks[i].page_cnt)
					<< PAGE_SHIFT));
				break;
			}
		}
	}
#endif
	for (i = 0; i < tz_mem_slot_size; i++) {
		if (_mtee_cma_in_range(&tz_mem_blocks[i], pfn_start, pfn_end) == false)
			continue;

		if (tz_mem_blocks[i].allocated == CMA_REGION_FREED)
			ret = false;

	}
	mutex_unlock(&tz_cma_allocate_lock);
	if (ret && size) {
		pr_info("%s 0x%x ~0x%0x, time = %d ms\n",
			__func__, (int)phy, (int)(pfn_end << PAGE_SHIFT),
			(int)(MTEE_GET_SYS_MS()-ms));
	}
	//if in pre-allocate mode, stay in cache
	if (ret == false && tz_cma_prealloc_buffer == false)
		_mtee_cma_free_impl_by_range(phy, size, false);

	return ret;
}

static bool  _mtee_cma_allocate(void)
{
	bool ret = false;

	if (tz_svp_memory_mode == TZPT_SVP_MEMORY_CARVEOUT) {
		pr_info("%s() should not called in svp carve-out memory mode\n", __func__);
		return false;
	}
	if (tz_svp_memory_mode == TZPT_SVP_MEMORY_RESERVED_MEMORY) {
		pr_info("%s() called in reserved memory mode\n", __func__);
		return true;
	}
	if (!tz_cma) {
		pr_info("%s() tz_cma is null !\n", __func__);
		return false;
	}
	if (tz_cma_prealloc_buffer == true)
		pr_info("pre-allocted buffer: %s: diff %d ms\n",
		__func__, (int)(MTEE_GET_SYS_MS()-tz_cma_prealloc_buffer_time));

	if (tz_cma_allocated) {
		pr_info("%s() already allocated!\n", __func__);
		return false;
	}
	mutex_lock(&tz_cma_api_lock);
#if defined(MTEE_CMA_DYNAMIC_MODE)
	//not need to allocate buffer in tz_cma_dynamic_mode, don't block flow
	if (tz_cma_dynamic_mode)
		ret = true;
	else
		ret = _mtee_cma_allocate_impl_by_range(tz_cma_base, tz_cma_secure_size, false);
#else
	ret = _mtee_cma_allocate_impl_by_range(tz_cma_base, tz_cma_secure_size, false);
#endif
	if (ret)
		tz_cma_allocated = true;
	mutex_unlock(&tz_cma_api_lock);
	return ret;
}

static bool _mtee_cma_free(void)
{
	bool ret = false;

	if (tz_svp_memory_mode == TZPT_SVP_MEMORY_CARVEOUT) {
		pr_info("%s() should not called in svp carve-out memory mode\n", __func__);
		return false;
	}
	if (tz_svp_memory_mode == TZPT_SVP_MEMORY_RESERVED_MEMORY) {
		pr_info("%s() called in reserved memory mode\n", __func__);
		return true;
	}
	if (!tz_cma) {
		pr_info("%s() tz_cma is null !\n", __func__);
		return false;
	}
	if (!tz_cma_allocated) {
		pr_info("%s() is not allocated !\n", __func__);
		return false;
	}
	mutex_lock(&tz_cma_api_lock);
#if defined(MTEE_CMA_CACHE_POLICY)
	ret = _mtee_cma_free_impl_by_range(tz_cma_base, tz_cma_secure_size, tz_cma_delay_free_ms ? true : false);
	if (tz_cma_delay_free_ms) {
		tz_last_free_called_ms = MTEE_GET_SYS_MS();
		schedule_delayed_work(&dleay_free_work,
			msecs_to_jiffies(MTEE_CMA_DELAY_FREE_POLLING_MS));
	}
#else
	ret = _mtee_cma_free_impl_by_range(tz_cma_base, tz_cma_secure_size, false);
#endif
	if (ret)
		tz_cma_allocated = false;
	mutex_unlock(&tz_cma_api_lock);

	return ret;
}

#if defined(MTEE_CMA_CACHE_POLICY)
static int tz_cma_delay_free_ms_show(struct seq_file *s, void *v)
{
	seq_printf(s, "%d\n", tz_cma_delay_free_ms);
	return 0;
}

static ssize_t tz_cma_delay_free_ms_proc_write(struct file *file,
				    const char __user *buffer,
				    size_t count, loff_t *pos)
{
	char buf[16] = {0};
	int new_mode = 0;

	if (count >= sizeof(buf)) {
		pr_info("%s: Invalid count\n", __func__);
		return -EINVAL;
	}
	if (copy_from_user(buf, buffer, count)) {
		pr_info("%s: copy_from_user failed\n", __func__);
		return -ENOMEM;
	}
	buf[sizeof(buf) - 1] = '\0';
	if (kstrtoint(buf, 10, &new_mode) < 0) {
		pr_info("%s: kstrtoint error\n", __func__);
		return -EINVAL;
	}
	tz_cma_delay_free_ms = new_mode;
	return count;
}
static int tz_cma_delay_free_ms_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, tz_cma_delay_free_ms_show, NULL);
}

static const struct proc_ops optee_cma_mode_proc_fops = {
	.proc_open		= tz_cma_delay_free_ms_proc_open,
	.proc_read		= seq_read,
	.proc_lseek		= seq_lseek,
	.proc_release	= single_release,
	.proc_write		= tz_cma_delay_free_ms_proc_write,
};

static int tz_cma_prealloc_show(struct seq_file *s, void *v)
{
	seq_printf(s, "%d\n", tz_cma_prealloc_buffer?1:0);
	return 0;
}

static int _mtee_cma_prealloc_buffer_task(void *data)
{
	size_t update_level = 0;

	for (;;) {
		if (wait_for_completion_timeout(&pre_alloc_task_start,
			msecs_to_jiffies(MTEE_CMA_COMPLETION_TIMEOUT_MS)) == 0)
			continue;
		do {
			update_level = 0;
			tz_cma_prealloc_buffer_time = MTEE_GET_SYS_MS();
			tz_cma_prealloc_buffer = true;
			while (tz_cma_cache_level != update_level) {
				update_level = tz_cma_cache_level;
				pr_info("pre-allocted buffer size: 0x%x\n",
					(int)update_level);
				_mtee_cma_allocate_impl_by_range(tz_cma_base, update_level, false);
			}
			tz_cma_prealloc_buffer = false;
			tz_cma_cache_level = 0;
		} while (0);
		reinit_completion(&pre_alloc_task_start);
	}
	return 0;
}

static ssize_t tz_cma_prealloc_proc_write(struct file *file,
				    const char __user *buffer,
				    size_t count, loff_t *pos)
{
	char buf[16] = {0};
	char *search_link_break = NULL;

	if (count >= sizeof(buf)) {
		pr_info("%s: Invalid count\n", __func__);
		return -EINVAL;
	}

	if (copy_from_user(buf, buffer, count)) {
		pr_info("%s: copy_from_user failed\n", __func__);
		return -ENOMEM;
	}
	buf[sizeof(buf) - 1] = '\0';
	search_link_break = strchr(buf, '\n');
	if (search_link_break)
		*search_link_break = '\0';
	if (strcmp(buf, "uhd") == 0) {
#if IS_ENABLED(CONFIG_MTEE_CMA_DEBUG)
		if (tz_svp_uhd_preallocate_ms == 0) {
			tz_svp_uhd_preallocate_ms = MTEE_GET_SYS_MS();
			pr_info(" %s trigger preload uhd", __func__);
		}
#endif
		mtee_cma_update_cache_level(MTEE_CMA_UHD_FB_SIZE);
	} else if (strcmp(buf, "1") == 0) {
		mutex_lock(&tz_cma_api_lock);
		tz_cma_cache_level = tz_cma_dynamic_mode ? MTEE_CMA_DEFAULT_SIZE : tz_cma_secure_size;
		if (pre_alloc_task_start.done == 0)
			complete(&pre_alloc_task_start);
		mutex_unlock(&tz_cma_api_lock);
	}
	return count;
}
static int tz_cma_prealloc_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, tz_cma_prealloc_show, NULL);
}

static const struct proc_ops optee_cma_pre_alloc_proc_fops = {
	.proc_open		= tz_cma_prealloc_proc_open,
	.proc_read		= seq_read,
	.proc_lseek		= seq_lseek,
	.proc_release	= single_release,
	.proc_write		= tz_cma_prealloc_proc_write,
};

static void  mtee_cma_update_cache_level(uint64_t size)
{
#if defined(MTEE_CMA_ENABLE_PARALLEL_ALLOC)
	int i = 0;
	size_t cached_size = 0;
	uint64_t phy = 0;
	uint64_t blk_size = 0;
	size_t update_level = 0;

	size = MTEE_CMA_ALIGN_UP_SIZE(size, MTEE_CMA_SLOT_SIZE_MB);

	if (tz_cma_map_region_size)
		size = MTEE_CMA_ALIGN_UP_SIZE(size, tz_cma_map_region_size);

	mutex_lock(&tz_cma_slot_lock);
	do {
		for (i = 0; i < tz_mem_slot_size; i++) {
			if (tz_mem_blocks[i].allocated == CMA_REGION_CACHED) {
				tz_mem_blocks[i].cached_time = MTEE_GET_SYS_MS();
				cached_size += tz_mem_blocks[i].page_cnt << PAGE_SHIFT;
			}
		}
		if (cached_size >= size)
			break;
		cached_size = 0;
		for (i = 0; i < tz_mem_slot_size; i++) {
			if (tz_mem_blocks[i].allocated != CMA_REGION_ALLOCATED) {
				phy = (uint64_t)tz_mem_blocks[i].pfn << PAGE_SHIFT;
				blk_size = (uint64_t)tz_mem_blocks[i].page_cnt << PAGE_SHIFT;
				cached_size += blk_size;
				update_level = (phy - tz_cma_base) + blk_size;
				if (cached_size >= size)
					break;
			}
		}
		if (update_level == 0)
			break;
		//align up to tz_cma_map_region_size
		if (tz_cma_map_region_size)
			update_level = MTEE_CMA_ALIGN_UP_SIZE(update_level, tz_cma_map_region_size);

		if (tz_cma_prealloc_buffer && update_level <= tz_cma_cache_level)
			break;

		tz_cma_cache_level = update_level;
		if (pre_alloc_task_start.done == 0)
			complete(&pre_alloc_task_start);
		pr_info("%s = %d MB\n", __func__, (int)(size/SZ_1M));
	} while (0);

	mutex_unlock(&tz_cma_slot_lock);
#endif
}
#endif

#if defined(MTEE_CMA_DYNAMIC_MODE)
static int tz_cma_dynamic_mode_show(struct seq_file *s, void *v)
{
	seq_printf(s, "%d\n", tz_cma_dynamic_mode ? 1 : 0);
	return 0;
}

static ssize_t tz_cma_dynamic_mode_proc_write(struct file *file,
				    const char __user *buffer,
				    size_t count, loff_t *pos)
{
	char buf[16] = {0};
	int new_mode = 0;

	if (count >= sizeof(buf)) {
		pr_info("%s: Invalid count\n", __func__);
		return -EINVAL;
	}

	if (copy_from_user(buf, buffer, count)) {
		pr_info("%s: copy_from_user failed\n", __func__);
		return -ENOMEM;
	}
	buf[sizeof(buf) - 1] = '\0';
	if (kstrtoint(buf, 10, &new_mode) < 0) {
		pr_info("%s: kstrtoint error\n", __func__);
		return -EINVAL;
	}
	if (tz_svp_memory_mode == TZPT_SVP_MEMORY_CMA)
		tz_cma_dynamic_mode = new_mode?true:false;

	return count;
}

static int tz_cma_dynamic_mode_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, tz_cma_dynamic_mode_show, NULL);
}

static const struct proc_ops optee_cma_dynamic_mode_proc_fops = {
	.proc_open		= tz_cma_dynamic_mode_proc_open,
	.proc_read		= seq_read,
	.proc_lseek		= seq_lseek,
	.proc_release	= single_release,
	.proc_write		= tz_cma_dynamic_mode_proc_write,
};
#endif

#if defined(MTEE_CMA_CACHE_POLICY)
static unsigned long tz_cm_shrink(int nr_to_scan, bool force_free)
{
	int only_scan = 0;
	int nr_total = 0;
	int i = 0;
	int total_cached = 0;
	int check_cache_page_cnt;
	int region_page_cnt = tz_cma_map_region_size >> PAGE_SHIFT;

	if (region_page_cnt)
		check_cache_page_cnt = MTEE_CMA_ALIGN_UP_SIZE(MTEE_CMA_CACHE_FOR_UHD_PAGE_CNT, region_page_cnt);
	else
		check_cache_page_cnt = MTEE_CMA_CACHE_FOR_UHD_PAGE_CNT;

	if (!nr_to_scan)
		only_scan = 1;

	do {
		if (!tz_cma_dynamic_mode)
			break;

		for (i = tz_mem_slot_size-1; i >= 0; i--) {
			if (tz_mem_blocks[i].allocated == CMA_REGION_CACHED)
				total_cached += tz_mem_blocks[i].page_cnt;
		}

		for (i = tz_mem_slot_size-1; i >= 0; i--) {
			if (tz_mem_blocks[i].allocated == CMA_REGION_CACHED) {

				if (tz_cma_prealloc_uhd_buffer && total_cached - nr_total <= check_cache_page_cnt)
					break;

				if (force_free == false &&
					(MTEE_GET_SYS_MS() - tz_mem_blocks[i].cached_time < tz_cma_delay_free_ms))
					continue;

				if (only_scan)
					nr_total += tz_mem_blocks[i].page_cnt;
				else {
					nr_total += tz_mem_blocks[i].page_cnt;
					free_contig_range(tz_mem_blocks[i].pfn, tz_mem_blocks[i].page_cnt);
					tz_mem_blocks[i].allocated = CMA_REGION_FREED;
					if (nr_total >= nr_to_scan)
						break;
				}
			}
		}
	} while (0);
	return nr_total;
}
static unsigned long tz_cm_count(struct shrinker *s,
						struct shrink_control *sc)
{
	unsigned long result = 0;

	if (mutex_trylock(&tz_cma_slot_lock) == 0)
		return 0;

	result = tz_cm_shrink(0, false);
	mutex_unlock(&tz_cma_slot_lock);
	return result;
}

static unsigned long tz_cm_scan(struct shrinker *s,
						struct shrink_control *sc)
{
	unsigned long result = 0;

	if (mutex_trylock(&tz_cma_allocate_lock) == 0)
		return SHRINK_STOP;

	if (mutex_trylock(&tz_cma_slot_lock) == 0) {
		mutex_unlock(&tz_cma_allocate_lock);
		return SHRINK_STOP;
	}
	result = tz_cm_shrink(sc->nr_to_scan, false);
	mutex_unlock(&tz_cma_slot_lock);
	mutex_unlock(&tz_cma_allocate_lock);
	return result;
}

static struct shrinker tz_cm_shrinker = {
	.scan_objects = tz_cm_scan,
	.count_objects = tz_cm_count,
	.seeks = DEFAULT_SEEKS
};

#endif

#if IS_ENABLED(CONFIG_MTEE_CMA_DEBUG)

static int tz_cma_debug_test_secure_buffer_show(struct seq_file *s, void *v)
{
	int i = 0;
	int j = 0;
	bool check_result = true;
	uint8_t *test_pattern;
	size_t check_size = 0;

	seq_puts(s, "cma buffer_check\n");
	test_pattern = vmalloc(1 << PAGE_SHIFT);
	if (!test_pattern) {
		seq_puts(s, "fail due to vmalloc is NULL\n");
		return -EINVAL;
	}
	memset(test_pattern, 0, 1 << PAGE_SHIFT);
	mutex_lock(&tz_cma_api_lock);
	for (i = 0; i < tz_mem_slot_size; i++) {
		if (tz_mem_blocks[i].allocated == CMA_REGION_ALLOCATED) {
			for (j = 0; j < tz_mem_blocks[i].page_cnt; j++) {
				uint8_t *va = (uint8_t *)__va((tz_mem_blocks[i].pfn + j) << PAGE_SHIFT);
				int ret;

				if (!va) {
					seq_printf(s, "get va fail, pa = %08x\n", (int)(int)(tz_mem_blocks[i].pfn + j));
					check_result = false;
					break;
				}
				ret = memcmp(va, test_pattern, 1 << PAGE_SHIFT);

				if (ret != 0) {
					check_result = false;
					seq_printf(s, "check svp buffer 0x%x ~0x%0x failed!!\n",
						(int)(tz_mem_blocks[i].pfn) << PAGE_SHIFT,
						(int)(tz_mem_blocks[i].pfn + tz_mem_blocks[i].page_cnt) << PAGE_SHIFT);
					break;
				}
			}
			if (check_result) {
				seq_printf(s, "check svp buffer 0x%x ~0x%0x pass!!\n",
						(int)(tz_mem_blocks[i].pfn) << PAGE_SHIFT,
						(int)(tz_mem_blocks[i].pfn + tz_mem_blocks[i].page_cnt) << PAGE_SHIFT);
				check_size += (tz_mem_blocks[i].page_cnt << PAGE_SHIFT);
			}
		}
	}
	mutex_unlock(&tz_cma_api_lock);
	if (check_result && check_size)
		seq_puts(s, "secure buffer check passed\n");
	else if (check_result == false)
		seq_puts(s, "secure buffer check failed\n");
	else
		seq_puts(s, "no secure buffer tested\n");
	vfree(test_pattern);
	return 0;
}

static ssize_t tz_cma_debug_test_secure_buffer_proc_write(struct file *file,
					const char __user *buffer,
					size_t count, loff_t *pos)
{
	return count;
}

static int tz_cma_debug_test_secure_buffer_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, tz_cma_debug_test_secure_buffer_show, NULL);
}

static const struct proc_ops optee_cma_debug_test_secure_buffer_proc_fops = {
	.proc_open		= tz_cma_debug_test_secure_buffer_proc_open,
	.proc_read		= seq_read,
	.proc_lseek		= seq_lseek,
	.proc_release	= single_release,
	.proc_write		= tz_cma_debug_test_secure_buffer_proc_write,
};


static int tz_cma_debug_max_memory_show(struct seq_file *s, void *v)
{
	seq_printf(s, "test svp fail cma on, upper bound = %d MB\n", tz_svp_debug_max_mem_mb);
	return 0;
}

static ssize_t tz_cma_debug_max_memory_proc_write(struct file *file,
					const char __user *buffer,
					size_t count, loff_t *pos)
{
	char buf[16] = {0};
	int max_mem_mb = 0;

	if (count >= sizeof(buf)) {
		pr_info("%s: Invalid count\n", __func__);
		return -EINVAL;
	}

	if (copy_from_user(buf, buffer, count)) {
		pr_info("%s: copy_from_user failed\n", __func__);
		return -ENOMEM;
	}
	buf[sizeof(buf) - 1] = '\0';
	if (kstrtoint(buf, 10, &max_mem_mb) < 0) {
		pr_info("%s: kstrtoint error\n", __func__);
		return -EINVAL;
	}
	if (max_mem_mb == -1)
		tz_svp_debug_max_mem_mb = 0xFFFFFFFF;
	else
		tz_svp_debug_max_mem_mb = max_mem_mb;

	return count;
}

static int tz_cma_debug_max_memory_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, tz_cma_debug_max_memory_show, NULL);
}

static const struct proc_ops optee_cma_debug_max_memory_proc_fops = {
	.proc_open		= tz_cma_debug_max_memory_proc_open,
	.proc_read		= seq_read,
	.proc_lseek		= seq_lseek,
	.proc_release	= single_release,
	.proc_write		= tz_cma_debug_max_memory_proc_write,
};

#endif

bool  mtee_cma_allocate(void)
{
#if IS_ENABLED(CONFIG_MTEE_CMA_DEBUG)
	if (tz_cma_dynamic_mode == false && tz_cma_secure_size >= tz_svp_debug_max_mem_mb * SZ_1M) {
		pr_info("%s test mode detect, return false", __func__);
		return false;
	}
#endif
	return _mtee_cma_allocate();
}

bool mtee_cma_free(void)
{
	return _mtee_cma_free();
}

bool mtee_cma_get_info(uint64_t *pa_base, uint64_t *size, uint8_t *dynamic_mode)
{
	*pa_base = tz_cma_base;
	*size = tz_cma_secure_size;
	#if defined(MTEE_CMA_DYNAMIC_MODE)
	*dynamic_mode = tz_cma_dynamic_mode ? 1 : 0;
	#else
	*dynamic_mode = 0;
	#endif
	return true;
}

bool  mtee_cma_map(uint64_t phy, uint64_t size)
{
	bool ret = true;
	unsigned long ms = MTEE_GET_SYS_MS();
	uint64_t next_cache_size = tz_cma_prealloc_uhd_buffer ? (MTEE_CMA_CACHE_FOR_UHD_MEM) : size;

#if IS_ENABLED(CONFIG_MTEE_CMA_DEBUG)
	if (phy - tz_cma_base >= (uint64_t)tz_svp_debug_max_mem_mb * SZ_1M) {
		pr_info("%s test mode detect, return false", __func__);
		return false;
	}
#endif
#if defined(MTEE_CMA_CACHE_POLICY)
	tz_cma_map_region_size = size;
#endif
	mutex_lock(&tz_cma_api_lock);
	ret =  _mtee_cma_allocate_impl_by_range(phy, size, true);
	if (ret)
		mtee_cma_update_cache_level(next_cache_size);
	mutex_unlock(&tz_cma_api_lock);
	pr_info("%s 0x%x ~0x%0x, time = %d ms\n",
		__func__, (int)phy, (int)(phy + size),
		(int)(MTEE_GET_SYS_MS()-ms));

	return ret;
}

bool  mtee_cma_unmap(uint64_t phy, uint64_t size)
{
	bool ret = true;
	unsigned long ms = MTEE_GET_SYS_MS();

	mutex_lock(&tz_cma_api_lock);
#if defined(MTEE_CMA_CACHE_POLICY)
	ret = _mtee_cma_free_impl_by_range(phy, size, true);
#else
	ret = _mtee_cma_free_impl_by_range(phy, size, false);
#endif
	mutex_unlock(&tz_cma_api_lock);
	pr_info("%s 0x%x ~0x%0x, time = %d ms\n",
		__func__, (int)phy, (int)(phy + size),
		(int)(MTEE_GET_SYS_MS()-ms));

	return ret;
}

int mtee_cma_init(struct platform_device *pdev)
{
	struct device_node *np;
	struct reserved_mem *rmem;
	struct proc_dir_entry *dir = NULL;
	int i = 0;
	unsigned long page_cnt;
	unsigned long last_page_cnt;
	#if defined(MTEE_CMA_CACHE_POLICY) || defined(MTEE_CMA_ENABLE_PARALLEL_ALLOC)
	struct task_struct *task;
	#endif

	/* get reverved memory address for secure wfd application  */
	np = of_parse_phandle(pdev->dev.of_node, "memory-region", 0);
	if (!np) {
		pr_info("No secure memory-region\n");
		return -EINVAL;
	}

	rmem = of_reserved_mem_lookup(np);
	if (!rmem) {
		pr_info("failed to get secure memory-region\n");
		return -EINVAL;
	}
	//add reference
	of_node_put(np);

	tz_cma_dev = pdev;
	tz_cma = (struct cma *)rmem->priv;
	tz_cma_secure_size = (size_t)rmem->size;
	tz_cma_base =  rmem->base;
	tz_svp_memory_mode = tz_cma ? TZPT_SVP_MEMORY_CMA : TZPT_SVP_MEMORY_RESERVED_MEMORY;
	tz_cma_allocated = false;
	#if defined(MTEE_CMA_CACHE_POLICY)
	tz_cma_delay_free_ms = MTEE_CMA_DELAY_FREE_CHECK_MS;
	tz_cma_prealloc_buffer = false;
	tz_cma_prealloc_buffer_time = 0;
	tz_cma_cache_level = MTEE_CMA_DEFAULT_SIZE;
	#endif
	mutex_init(&tz_cma_api_lock);
	mutex_init(&tz_cma_allocate_lock);

	#if defined(MTEE_CMA_ENABLE_PARALLEL_ALLOC)
	mutex_init(&tz_cma_slot_lock);
	#endif
	#if defined(MTEE_CMA_DYNAMIC_MODE)
	if (tz_svp_memory_mode == TZPT_SVP_MEMORY_RESERVED_MEMORY)
		tz_cma_dynamic_mode = false;
	else
		tz_cma_dynamic_mode = true;
	#endif

	tz_mem_slot_size = (((tz_cma_secure_size/SZ_1M) + (MTEE_CMA_SLOT_SIZE_MB-1))/
						MTEE_CMA_SLOT_SIZE_MB);
	tz_mem_blocks = vmalloc(sizeof(struct tz_mem_unit) * tz_mem_slot_size);
	if (tz_mem_blocks == NULL) {
		pr_info("failed to vmalloc tz_mem_blocks\n");
		return -EINVAL;
	}
	memset(tz_mem_blocks, 0, (sizeof(struct tz_mem_unit) * tz_mem_slot_size));

	page_cnt = ((MTEE_CMA_SLOT_SIZE_MB * SZ_1M) >> PAGE_SHIFT);
	last_page_cnt = (tz_cma_secure_size  >> PAGE_SHIFT)
					- page_cnt * (tz_mem_slot_size-1);
	for (i = 0; i < tz_mem_slot_size; i++) {
		tz_mem_blocks[i].pfn = PFN_DOWN(tz_cma_base) + (page_cnt) * i;
		tz_mem_blocks[i].allocated = CMA_REGION_FREED;
		if (i != (tz_mem_slot_size - 1))
			tz_mem_blocks[i].page_cnt = page_cnt;
		else
			tz_mem_blocks[i].page_cnt = last_page_cnt;
	}
	#if defined(MTEE_CMA_ENABLE_PARALLEL_ALLOC)
	memset(&parallel_tasks, 0, sizeof(parallel_tasks));
	for (i = 0; i < MTEE_CMA_PARALLEL_TASK_NUM; i++) {
		init_completion(&parallel_tasks[i].task_start);
		init_completion(&parallel_tasks[i].task_finish);
		task = kthread_create(_mtee_cma_parallel_alloc_task,
							(void *)&parallel_tasks[i],
							"parrel_cma_%d",
							i);
		set_user_nice(task, MTEE_CMA_PARALLEL_TASK_USER_NICE);
		wake_up_process(task);
	}
	#endif

	#if defined(MTEE_CMA_CACHE_POLICY)
	INIT_DELAYED_WORK(&dleay_free_work, _mtee_cma_check_free);
	init_completion(&pre_alloc_task_start);
	tz_cma_prealloc_uhd_buffer = false;
	tz_cma_map_region_size = 0;
	tz_cma_first4k_buffer_handle = -1;
	task = kthread_create(_mtee_cma_prealloc_buffer_task,
									NULL,
									"%s",
									"cma_prealloc_buffer");
	if (task)
		wake_up_process(task);
	dir = proc_mkdir("cma_svp", NULL);
	if (!proc_create_data("delay_free_ms", 0660, dir, &optee_cma_mode_proc_fops,
			       NULL)) {
		pr_info("create /proc/cma_svp/delay_free_ms fails!\n");
		return -1;
	}
	if (!proc_create_data("pre_alloc", 0660, dir, &optee_cma_pre_alloc_proc_fops,
			       NULL)) {
		pr_info("create /proc/cma_svp/prealloc_start fails!\n");
		return -1;
	}
	#if defined(MTEE_CMA_DYNAMIC_MODE)
	if (!proc_create_data("dynamic_mode", 0660, dir, &optee_cma_dynamic_mode_proc_fops,
			       NULL)) {
		pr_info("create /proc/cma_svp/dynamic_mode fails!\n");
		return -1;
	}
	#endif
	register_shrinker(&tz_cm_shrinker);
	#if IS_ENABLED(CONFIG_MTEE_CMA_DEBUG)
	tz_svp_debug_max_mem_mb = 0xFFFFFFFF;
	tz_svp_uhd_preallocate_ms = 0;

	if (!proc_create_data("test_secure_buffer", 0660, dir, &optee_cma_debug_test_secure_buffer_proc_fops,
			       NULL)) {
		pr_info("create /proc/cma_svp/test_secure_buffer fails!\n");
	}
	if (!proc_create_data("max_memory_mb", 0660, dir, &optee_cma_debug_max_memory_proc_fops,
			       NULL)) {
		pr_info("create /proc/cma_svp/max_memory_mb fails!\n");
	}
	#endif
	#endif

	return 0;
}

bool mtee_cma_preallocate_buffer_check(KREE_SECUREMEM_HANDLE cm_handle, uint32_t size, bool alloc)
{
#if defined(MTEE_CMA_CACHE_POLICY)
	if (alloc) {
		if (size >= MTEE_CMA_UHD_BUFFER_CHECK_SIZE) {
			if (tz_cma_first4k_buffer_handle == -1) {
#if IS_ENABLED(CONFIG_MTEE_CMA_DEBUG)
				pr_info(" %s uhd diff = %d\n", __func__, (int)(MTEE_GET_SYS_MS() - tz_svp_uhd_preallocate_ms));
				tz_svp_uhd_preallocate_ms = 0;
#endif
				mutex_lock(&tz_cma_api_lock);
				mtee_cma_update_cache_level(size*MTEE_CMA_UHD_FB_NUM);
				mutex_unlock(&tz_cma_api_lock);
			}
			tz_cma_first4k_buffer_handle = cm_handle;
			tz_cma_prealloc_uhd_buffer = false;
		} else if (tz_cma_first4k_buffer_handle == -1)
			tz_cma_prealloc_uhd_buffer = true;
	} else {
		if (tz_cma_first4k_buffer_handle == cm_handle) {
			tz_cma_first4k_buffer_handle = -1;
			tz_cma_prealloc_uhd_buffer = true;
		}
	}
#endif
	return true;
}