midgard/r12p0/kernel/drivers/gpu/arm/midgard/mali_kbase_js_policy_cfs.c - manifest_repos/mali-driver - Git at Google

 /*
  *
  * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
  * Foundation, and any use by you of this program is subject to the terms
  * of such GNU licence.
  *
  * A copy of the licence is included with the program, and can also be obtained
  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  * Boston, MA  02110-1301, USA.
  *
  */


 /*
  * Job Scheduler: Completely Fair Policy Implementation
  */

 #include <mali_kbase.h>
 #include <mali_kbase_js.h>
 #include <mali_kbase_js_policy_cfs.h>
 #include <linux/version.h>
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
 #include <linux/sched/rt.h>
 #endif

 /**
  * Define for when dumping is enabled.
  * This should not be based on the instrumentation level as whether dumping is enabled for a particular level is down to the integrator.
  * However this is being used for now as otherwise the cinstr headers would be needed.
  */
 #define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL)

 /* Fixed point constants used for runtime weight calculations */
 #define WEIGHT_FIXEDPOINT_SHIFT 10
 #define WEIGHT_TABLE_SIZE       40
 #define WEIGHT_0_NICE           (WEIGHT_TABLE_SIZE/2)
 #define WEIGHT_0_VAL            (1 << WEIGHT_FIXEDPOINT_SHIFT)

 #define PROCESS_PRIORITY_MIN (-20)
 #define PROCESS_PRIORITY_MAX  (19)

 /* Defines for easy asserts 'is scheduled'/'is queued'/'is neither queued norscheduled' */
 #define KBASEP_JS_CHECKFLAG_QUEUED       (1u << 0) /**< Check the queued state */
 #define KBASEP_JS_CHECKFLAG_SCHEDULED    (1u << 1) /**< Check the scheduled state */
 #define KBASEP_JS_CHECKFLAG_IS_QUEUED    (1u << 2) /**< Expect queued state to be set */
 #define KBASEP_JS_CHECKFLAG_IS_SCHEDULED (1u << 3) /**< Expect scheduled state to be set */

 enum {
 	KBASEP_JS_CHECK_NOTQUEUED = KBASEP_JS_CHECKFLAG_QUEUED,
 	KBASEP_JS_CHECK_NOTSCHEDULED = KBASEP_JS_CHECKFLAG_SCHEDULED,
 	KBASEP_JS_CHECK_QUEUED = KBASEP_JS_CHECKFLAG_QUEUED | KBASEP_JS_CHECKFLAG_IS_QUEUED,
 	KBASEP_JS_CHECK_SCHEDULED = KBASEP_JS_CHECKFLAG_SCHEDULED | KBASEP_JS_CHECKFLAG_IS_SCHEDULED
 };

 typedef u32 kbasep_js_check;

 /*
  * Private Functions
  */

 /* Table autogenerated using util built from: base/tools/gen_cfs_weight_of_prio/ */

 /* weight = 1.25 */
 static const int weight_of_priority[] = {
 	/*  -20 */ 11, 14, 18, 23,
 	/*  -16 */ 29, 36, 45, 56,
 	/*  -12 */ 70, 88, 110, 137,
 	/*   -8 */ 171, 214, 268, 335,
 	/*   -4 */ 419, 524, 655, 819,
 	/*    0 */ 1024, 1280, 1600, 2000,
 	/*    4 */ 2500, 3125, 3906, 4883,
 	/*    8 */ 6104, 7630, 9538, 11923,
 	/*   12 */ 14904, 18630, 23288, 29110,
 	/*   16 */ 36388, 45485, 56856, 71070
 };

 /*
  * Note: There is nothing to stop the priority of the ctx containing
  * ctx_info changing during or immediately after this function is called
  * (because its jsctx_mutex cannot be held during IRQ). Therefore, this
  * function should only be seen as a heuristic guide as to the priority weight
  * of the context.
  */
 static u64 priority_weight(struct kbasep_js_policy_cfs_ctx *ctx_info, u64 time_us)
 {
 	u64 time_delta_us;
 	int priority;

 	priority = ctx_info->process_priority;

 	/* Adjust runtime_us using priority weight if required */
 	if (priority != 0 && time_us != 0) {
 		int clamped_priority;

 		/* Clamp values to min..max weights */
 		if (priority > PROCESS_PRIORITY_MAX)
 			clamped_priority = PROCESS_PRIORITY_MAX;
 		else if (priority < PROCESS_PRIORITY_MIN)
 			clamped_priority = PROCESS_PRIORITY_MIN;
 		else
 			clamped_priority = priority;

 		/* Fixed point multiplication */
 		time_delta_us = (time_us * weight_of_priority[WEIGHT_0_NICE + clamped_priority]);
 		/* Remove fraction */
 		time_delta_us = time_delta_us >> WEIGHT_FIXEDPOINT_SHIFT;
 		/* Make sure the time always increases */
 		if (0 == time_delta_us)
 			time_delta_us++;
 	} else {
 		time_delta_us = time_us;
 	}

 	return time_delta_us;
 }

 #if KBASE_TRACE_ENABLE
 static int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device *kbdev, struct kbase_context *kctx)
 {
 	struct kbasep_js_device_data *js_devdata;
 	int as_nr;
 	int refcnt = 0;

 	js_devdata = &kbdev->js_data;

 	as_nr = kctx->as_nr;
 	if (as_nr != KBASEP_AS_NR_INVALID) {
 		struct kbasep_js_per_as_data *js_per_as_data;

 		js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];

 		refcnt = js_per_as_data->as_busy_refcount;
 	}

 	return refcnt;
 }

 static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
 {
 	unsigned long flags;
 	struct kbasep_js_device_data *js_devdata;
 	int refcnt = 0;

 	js_devdata = &kbdev->js_data;

 	spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
 	refcnt = kbasep_js_policy_trace_get_refcnt_nolock(kbdev, kctx);
 	spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);

 	return refcnt;
 }
 #else				/* KBASE_TRACE_ENABLE  */
 static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
 {
 	CSTD_UNUSED(kbdev);
 	CSTD_UNUSED(kctx);
 	return 0;
 }
 #endif				/* KBASE_TRACE_ENABLE  */


 /*
  * Non-private functions
  */

 int kbasep_js_policy_init(struct kbase_device *kbdev)
 {
 	struct kbasep_js_device_data *js_devdata;
 	struct kbasep_js_policy_cfs *policy_info;

 	KBASE_DEBUG_ASSERT(kbdev != NULL);
 	js_devdata = &kbdev->js_data;
 	policy_info = &js_devdata->policy.cfs;

 	atomic64_set(&policy_info->least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
 	atomic64_set(&policy_info->rt_least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);

 	policy_info->head_runtime_us = 0;

 	return 0;
 }

 void kbasep_js_policy_term(union kbasep_js_policy *js_policy)
 {
 	CSTD_UNUSED(js_policy);
 }

 int kbasep_js_policy_init_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
 {
 	struct kbasep_js_device_data *js_devdata;
 	struct kbasep_js_policy_cfs_ctx *ctx_info;
 	struct kbasep_js_policy_cfs *policy_info;
 	int policy;

 	KBASE_DEBUG_ASSERT(kbdev != NULL);
 	KBASE_DEBUG_ASSERT(kctx != NULL);

 	js_devdata = &kbdev->js_data;
 	policy_info = &kbdev->js_data.policy.cfs;
 	ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;

 	KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_INIT_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));

 	policy = current->policy;
 	if (policy == SCHED_FIFO || policy == SCHED_RR) {
 		ctx_info->process_rt_policy = true;
 		ctx_info->process_priority = (((MAX_RT_PRIO - 1) - current->rt_priority) / 5) - 20;
 	} else {
 		ctx_info->process_rt_policy = false;
 		ctx_info->process_priority = (current->static_prio - MAX_RT_PRIO) - 20;
 	}

 	/* Initial runtime (relative to least-run context runtime)
 	 *
 	 * This uses the Policy Queue's most up-to-date head_runtime_us by using the
 	 * queue mutex to issue memory barriers - also ensure future updates to
 	 * head_runtime_us occur strictly after this context is initialized */
 	mutex_lock(&js_devdata->queue_mutex);

 	/* No need to hold the the runpool_irq.lock here, because we're initializing
 	 * the value, and the context is definitely not being updated in the
 	 * runpool at this point. The queue_mutex ensures the memory barrier. */
 	ctx_info->runtime_us = policy_info->head_runtime_us + priority_weight(ctx_info, (u64) js_devdata->cfs_ctx_runtime_init_slices * (u64) (js_devdata->ctx_timeslice_ns / 1000u));

 	mutex_unlock(&js_devdata->queue_mutex);

 	return 0;
 }

 void kbasep_js_policy_term_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
 {
 	struct kbasep_js_policy_cfs_ctx *ctx_info;
 	struct kbasep_js_policy_cfs *policy_info;
 	struct kbase_device *kbdev;

 	KBASE_DEBUG_ASSERT(js_policy != NULL);
 	KBASE_DEBUG_ASSERT(kctx != NULL);

 	policy_info = &js_policy->cfs;
 	ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;

 	kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
 	KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_TERM_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));

 	/* No work to do */
 }

 /*
  * Job Chain Management
  */

 void kbasep_js_policy_log_job_result(union kbasep_js_policy *js_policy, struct kbase_jd_atom *katom, u64 time_spent_us)
 {
 	struct kbasep_js_policy_cfs_ctx *ctx_info;
 	struct kbase_context *parent_ctx;

 	KBASE_DEBUG_ASSERT(js_policy != NULL);
 	KBASE_DEBUG_ASSERT(katom != NULL);
 	CSTD_UNUSED(js_policy);

 	parent_ctx = katom->kctx;
 	KBASE_DEBUG_ASSERT(parent_ctx != NULL);

 	ctx_info = &parent_ctx->jctx.sched_info.runpool.policy_ctx.cfs;

 	ctx_info->runtime_us += priority_weight(ctx_info, time_spent_us);

 	katom->time_spent_us += time_spent_us;
 }

 bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy *js_policy, struct kbase_context *current_ctx, struct kbase_context *new_ctx)
 {
 	struct kbasep_js_policy_cfs_ctx *current_ctx_info;
 	struct kbasep_js_policy_cfs_ctx *new_ctx_info;

 	KBASE_DEBUG_ASSERT(current_ctx != NULL);
 	KBASE_DEBUG_ASSERT(new_ctx != NULL);
 	CSTD_UNUSED(js_policy);

 	current_ctx_info = &current_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
 	new_ctx_info = &new_ctx->jctx.sched_info.runpool.policy_ctx.cfs;

 	if (!current_ctx_info->process_rt_policy && new_ctx_info->process_rt_policy)
 		return true;

 	if (current_ctx_info->process_rt_policy ==
 			new_ctx_info->process_rt_policy)
 		return true;

 	return false;
 }
	/*
	*
	* (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
	*
	* This program is free software and is provided to you under the terms of the
	* GNU General Public License version 2 as published by the Free Software
	* Foundation, and any use by you of this program is subject to the terms
	* of such GNU licence.
	*
	* A copy of the licence is included with the program, and can also be obtained
	* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
	* Boston, MA 02110-1301, USA.
	*
	*/





	/*
	* Job Scheduler: Completely Fair Policy Implementation
	*/

	#include <mali_kbase.h>
	#include <mali_kbase_js.h>
	#include <mali_kbase_js_policy_cfs.h>
	#include <linux/version.h>
	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
	#include <linux/sched/rt.h>
	#endif

	/**
	* Define for when dumping is enabled.
	* This should not be based on the instrumentation level as whether dumping is enabled for a particular level is down to the integrator.
	* However this is being used for now as otherwise the cinstr headers would be needed.
	*/
	#define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL)

	/* Fixed point constants used for runtime weight calculations */
	#define WEIGHT_FIXEDPOINT_SHIFT 10
	#define WEIGHT_TABLE_SIZE 40
	#define WEIGHT_0_NICE (WEIGHT_TABLE_SIZE/2)
	#define WEIGHT_0_VAL (1 << WEIGHT_FIXEDPOINT_SHIFT)

	#define PROCESS_PRIORITY_MIN (-20)
	#define PROCESS_PRIORITY_MAX (19)

	/* Defines for easy asserts 'is scheduled'/'is queued'/'is neither queued norscheduled' */
	#define KBASEP_JS_CHECKFLAG_QUEUED (1u << 0) /*< Check the queued state /
	#define KBASEP_JS_CHECKFLAG_SCHEDULED (1u << 1) /*< Check the scheduled state /
	#define KBASEP_JS_CHECKFLAG_IS_QUEUED (1u << 2) /*< Expect queued state to be set /
	#define KBASEP_JS_CHECKFLAG_IS_SCHEDULED (1u << 3) /*< Expect scheduled state to be set /

	enum {
	KBASEP_JS_CHECK_NOTQUEUED = KBASEP_JS_CHECKFLAG_QUEUED,
	KBASEP_JS_CHECK_NOTSCHEDULED = KBASEP_JS_CHECKFLAG_SCHEDULED,
	KBASEP_JS_CHECK_QUEUED = KBASEP_JS_CHECKFLAG_QUEUED \| KBASEP_JS_CHECKFLAG_IS_QUEUED,
	KBASEP_JS_CHECK_SCHEDULED = KBASEP_JS_CHECKFLAG_SCHEDULED \| KBASEP_JS_CHECKFLAG_IS_SCHEDULED
	};

	typedef u32 kbasep_js_check;

	/*
	* Private Functions
	*/

	/* Table autogenerated using util built from: base/tools/gen_cfs_weight_of_prio/ */

	/* weight = 1.25 */
	static const int weight_of_priority[] = {
	/* -20 */ 11, 14, 18, 23,
	/* -16 */ 29, 36, 45, 56,
	/* -12 */ 70, 88, 110, 137,
	/* -8 */ 171, 214, 268, 335,
	/* -4 */ 419, 524, 655, 819,
	/* 0 */ 1024, 1280, 1600, 2000,
	/* 4 */ 2500, 3125, 3906, 4883,
	/* 8 */ 6104, 7630, 9538, 11923,
	/* 12 */ 14904, 18630, 23288, 29110,
	/* 16 */ 36388, 45485, 56856, 71070
	};

	/*
	* Note: There is nothing to stop the priority of the ctx containing
	* ctx_info changing during or immediately after this function is called
	* (because its jsctx_mutex cannot be held during IRQ). Therefore, this
	* function should only be seen as a heuristic guide as to the priority weight
	* of the context.
	*/
	static u64 priority_weight(struct kbasep_js_policy_cfs_ctx *ctx_info, u64 time_us)
	{
	u64 time_delta_us;
	int priority;

	priority = ctx_info->process_priority;

	/* Adjust runtime_us using priority weight if required */
	if (priority != 0 && time_us != 0) {
	int clamped_priority;

	/* Clamp values to min..max weights */
	if (priority > PROCESS_PRIORITY_MAX)
	clamped_priority = PROCESS_PRIORITY_MAX;
	else if (priority < PROCESS_PRIORITY_MIN)
	clamped_priority = PROCESS_PRIORITY_MIN;
	else
	clamped_priority = priority;

	/* Fixed point multiplication */
	time_delta_us = (time_us * weight_of_priority[WEIGHT_0_NICE + clamped_priority]);
	/* Remove fraction */
	time_delta_us = time_delta_us >> WEIGHT_FIXEDPOINT_SHIFT;
	/* Make sure the time always increases */
	if (0 == time_delta_us)
	time_delta_us++;
	} else {
	time_delta_us = time_us;
	}

	return time_delta_us;
	}

	#if KBASE_TRACE_ENABLE
	static int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device kbdev, struct kbase_context kctx)
	{
	struct kbasep_js_device_data *js_devdata;
	int as_nr;
	int refcnt = 0;

	js_devdata = &kbdev->js_data;

	as_nr = kctx->as_nr;
	if (as_nr != KBASEP_AS_NR_INVALID) {
	struct kbasep_js_per_as_data *js_per_as_data;

	js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];

	refcnt = js_per_as_data->as_busy_refcount;
	}

	return refcnt;
	}

	static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device kbdev, struct kbase_context kctx)
	{
	unsigned long flags;
	struct kbasep_js_device_data *js_devdata;
	int refcnt = 0;

	js_devdata = &kbdev->js_data;

	spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
	refcnt = kbasep_js_policy_trace_get_refcnt_nolock(kbdev, kctx);
	spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);

	return refcnt;
	}
	#else /* KBASE_TRACE_ENABLE */
	static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device kbdev, struct kbase_context kctx)
	{
	CSTD_UNUSED(kbdev);
	CSTD_UNUSED(kctx);
	return 0;
	}
	#endif /* KBASE_TRACE_ENABLE */


	/*
	* Non-private functions
	*/

	int kbasep_js_policy_init(struct kbase_device *kbdev)
	{
	struct kbasep_js_device_data *js_devdata;
	struct kbasep_js_policy_cfs *policy_info;

	KBASE_DEBUG_ASSERT(kbdev != NULL);
	js_devdata = &kbdev->js_data;
	policy_info = &js_devdata->policy.cfs;

	atomic64_set(&policy_info->least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
	atomic64_set(&policy_info->rt_least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);

	policy_info->head_runtime_us = 0;

	return 0;
	}

	void kbasep_js_policy_term(union kbasep_js_policy *js_policy)
	{
	CSTD_UNUSED(js_policy);
	}

	int kbasep_js_policy_init_ctx(struct kbase_device kbdev, struct kbase_context kctx)
	{
	struct kbasep_js_device_data *js_devdata;
	struct kbasep_js_policy_cfs_ctx *ctx_info;
	struct kbasep_js_policy_cfs *policy_info;
	int policy;

	KBASE_DEBUG_ASSERT(kbdev != NULL);
	KBASE_DEBUG_ASSERT(kctx != NULL);

	js_devdata = &kbdev->js_data;
	policy_info = &kbdev->js_data.policy.cfs;
	ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;

	KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_INIT_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));

	policy = current->policy;
	if (policy == SCHED_FIFO \|\| policy == SCHED_RR) {
	ctx_info->process_rt_policy = true;
	ctx_info->process_priority = (((MAX_RT_PRIO - 1) - current->rt_priority) / 5) - 20;
	} else {
	ctx_info->process_rt_policy = false;
	ctx_info->process_priority = (current->static_prio - MAX_RT_PRIO) - 20;
	}

	/* Initial runtime (relative to least-run context runtime)
	*
	* This uses the Policy Queue's most up-to-date head_runtime_us by using the
	* queue mutex to issue memory barriers - also ensure future updates to
	* head_runtime_us occur strictly after this context is initialized */
	mutex_lock(&js_devdata->queue_mutex);

	/* No need to hold the the runpool_irq.lock here, because we're initializing
	* the value, and the context is definitely not being updated in the
	* runpool at this point. The queue_mutex ensures the memory barrier. */
	ctx_info->runtime_us = policy_info->head_runtime_us + priority_weight(ctx_info, (u64) js_devdata->cfs_ctx_runtime_init_slices * (u64) (js_devdata->ctx_timeslice_ns / 1000u));

	mutex_unlock(&js_devdata->queue_mutex);

	return 0;
	}

	void kbasep_js_policy_term_ctx(union kbasep_js_policy js_policy, struct kbase_context kctx)
	{
	struct kbasep_js_policy_cfs_ctx *ctx_info;
	struct kbasep_js_policy_cfs *policy_info;
	struct kbase_device *kbdev;

	KBASE_DEBUG_ASSERT(js_policy != NULL);
	KBASE_DEBUG_ASSERT(kctx != NULL);

	policy_info = &js_policy->cfs;
	ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;

	kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
	KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_TERM_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));

	/* No work to do */
	}

	/*
	* Job Chain Management
	*/

	void kbasep_js_policy_log_job_result(union kbasep_js_policy js_policy, struct kbase_jd_atom katom, u64 time_spent_us)
	{
	struct kbasep_js_policy_cfs_ctx *ctx_info;
	struct kbase_context *parent_ctx;

	KBASE_DEBUG_ASSERT(js_policy != NULL);
	KBASE_DEBUG_ASSERT(katom != NULL);
	CSTD_UNUSED(js_policy);

	parent_ctx = katom->kctx;
	KBASE_DEBUG_ASSERT(parent_ctx != NULL);

	ctx_info = &parent_ctx->jctx.sched_info.runpool.policy_ctx.cfs;

	ctx_info->runtime_us += priority_weight(ctx_info, time_spent_us);

	katom->time_spent_us += time_spent_us;
	}

	bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy js_policy, struct kbase_context current_ctx, struct kbase_context *new_ctx)
	{
	struct kbasep_js_policy_cfs_ctx *current_ctx_info;
	struct kbasep_js_policy_cfs_ctx *new_ctx_info;

	KBASE_DEBUG_ASSERT(current_ctx != NULL);
	KBASE_DEBUG_ASSERT(new_ctx != NULL);
	CSTD_UNUSED(js_policy);

	current_ctx_info = &current_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
	new_ctx_info = &new_ctx->jctx.sched_info.runpool.policy_ctx.cfs;

	if (!current_ctx_info->process_rt_policy && new_ctx_info->process_rt_policy)
	return true;

	if (current_ctx_info->process_rt_policy ==
	new_ctx_info->process_rt_policy)
	return true;

	return false;
	}