| /* |
| * |
| * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. |
| * |
| * This program is free software and is provided to you under the terms of the |
| * GNU General Public License version 2 as published by the Free Software |
| * Foundation, and any use by you of this program is subject to the terms |
| * of such GNU licence. |
| * |
| * A copy of the licence is included with the program, and can also be obtained |
| * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, |
| * Boston, MA 02110-1301, USA. |
| * |
| */ |
| |
| |
| |
| |
| |
| /* |
| * Job Scheduler: Completely Fair Policy Implementation |
| */ |
| |
| #include <mali_kbase.h> |
| #include <mali_kbase_js.h> |
| #include <mali_kbase_js_policy_cfs.h> |
| #include <linux/version.h> |
| #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) |
| #include <linux/sched/rt.h> |
| #endif |
| |
| /** |
| * Define for when dumping is enabled. |
| * This should not be based on the instrumentation level as whether dumping is enabled for a particular level is down to the integrator. |
| * However this is being used for now as otherwise the cinstr headers would be needed. |
| */ |
| #define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL) |
| |
| /* Fixed point constants used for runtime weight calculations */ |
| #define WEIGHT_FIXEDPOINT_SHIFT 10 |
| #define WEIGHT_TABLE_SIZE 40 |
| #define WEIGHT_0_NICE (WEIGHT_TABLE_SIZE/2) |
| #define WEIGHT_0_VAL (1 << WEIGHT_FIXEDPOINT_SHIFT) |
| |
| #define PROCESS_PRIORITY_MIN (-20) |
| #define PROCESS_PRIORITY_MAX (19) |
| |
| /* Defines for easy asserts 'is scheduled'/'is queued'/'is neither queued norscheduled' */ |
| #define KBASEP_JS_CHECKFLAG_QUEUED (1u << 0) /**< Check the queued state */ |
| #define KBASEP_JS_CHECKFLAG_SCHEDULED (1u << 1) /**< Check the scheduled state */ |
| #define KBASEP_JS_CHECKFLAG_IS_QUEUED (1u << 2) /**< Expect queued state to be set */ |
| #define KBASEP_JS_CHECKFLAG_IS_SCHEDULED (1u << 3) /**< Expect scheduled state to be set */ |
| |
| enum { |
| KBASEP_JS_CHECK_NOTQUEUED = KBASEP_JS_CHECKFLAG_QUEUED, |
| KBASEP_JS_CHECK_NOTSCHEDULED = KBASEP_JS_CHECKFLAG_SCHEDULED, |
| KBASEP_JS_CHECK_QUEUED = KBASEP_JS_CHECKFLAG_QUEUED | KBASEP_JS_CHECKFLAG_IS_QUEUED, |
| KBASEP_JS_CHECK_SCHEDULED = KBASEP_JS_CHECKFLAG_SCHEDULED | KBASEP_JS_CHECKFLAG_IS_SCHEDULED |
| }; |
| |
| typedef u32 kbasep_js_check; |
| |
| /* |
| * Private Functions |
| */ |
| |
| /* Table autogenerated using util built from: base/tools/gen_cfs_weight_of_prio/ */ |
| |
| /* weight = 1.25 */ |
| static const int weight_of_priority[] = { |
| /* -20 */ 11, 14, 18, 23, |
| /* -16 */ 29, 36, 45, 56, |
| /* -12 */ 70, 88, 110, 137, |
| /* -8 */ 171, 214, 268, 335, |
| /* -4 */ 419, 524, 655, 819, |
| /* 0 */ 1024, 1280, 1600, 2000, |
| /* 4 */ 2500, 3125, 3906, 4883, |
| /* 8 */ 6104, 7630, 9538, 11923, |
| /* 12 */ 14904, 18630, 23288, 29110, |
| /* 16 */ 36388, 45485, 56856, 71070 |
| }; |
| |
| /* |
| * Note: There is nothing to stop the priority of the ctx containing |
| * ctx_info changing during or immediately after this function is called |
| * (because its jsctx_mutex cannot be held during IRQ). Therefore, this |
| * function should only be seen as a heuristic guide as to the priority weight |
| * of the context. |
| */ |
| static u64 priority_weight(struct kbasep_js_policy_cfs_ctx *ctx_info, u64 time_us) |
| { |
| u64 time_delta_us; |
| int priority; |
| |
| priority = ctx_info->process_priority; |
| |
| /* Adjust runtime_us using priority weight if required */ |
| if (priority != 0 && time_us != 0) { |
| int clamped_priority; |
| |
| /* Clamp values to min..max weights */ |
| if (priority > PROCESS_PRIORITY_MAX) |
| clamped_priority = PROCESS_PRIORITY_MAX; |
| else if (priority < PROCESS_PRIORITY_MIN) |
| clamped_priority = PROCESS_PRIORITY_MIN; |
| else |
| clamped_priority = priority; |
| |
| /* Fixed point multiplication */ |
| time_delta_us = (time_us * weight_of_priority[WEIGHT_0_NICE + clamped_priority]); |
| /* Remove fraction */ |
| time_delta_us = time_delta_us >> WEIGHT_FIXEDPOINT_SHIFT; |
| /* Make sure the time always increases */ |
| if (0 == time_delta_us) |
| time_delta_us++; |
| } else { |
| time_delta_us = time_us; |
| } |
| |
| return time_delta_us; |
| } |
| |
| #if KBASE_TRACE_ENABLE |
| static int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device *kbdev, struct kbase_context *kctx) |
| { |
| struct kbasep_js_device_data *js_devdata; |
| int as_nr; |
| int refcnt = 0; |
| |
| js_devdata = &kbdev->js_data; |
| |
| as_nr = kctx->as_nr; |
| if (as_nr != KBASEP_AS_NR_INVALID) { |
| struct kbasep_js_per_as_data *js_per_as_data; |
| |
| js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr]; |
| |
| refcnt = js_per_as_data->as_busy_refcount; |
| } |
| |
| return refcnt; |
| } |
| |
| static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx) |
| { |
| unsigned long flags; |
| struct kbasep_js_device_data *js_devdata; |
| int refcnt = 0; |
| |
| js_devdata = &kbdev->js_data; |
| |
| spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags); |
| refcnt = kbasep_js_policy_trace_get_refcnt_nolock(kbdev, kctx); |
| spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); |
| |
| return refcnt; |
| } |
| #else /* KBASE_TRACE_ENABLE */ |
| static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx) |
| { |
| CSTD_UNUSED(kbdev); |
| CSTD_UNUSED(kctx); |
| return 0; |
| } |
| #endif /* KBASE_TRACE_ENABLE */ |
| |
| |
| /* |
| * Non-private functions |
| */ |
| |
| int kbasep_js_policy_init(struct kbase_device *kbdev) |
| { |
| struct kbasep_js_device_data *js_devdata; |
| struct kbasep_js_policy_cfs *policy_info; |
| |
| KBASE_DEBUG_ASSERT(kbdev != NULL); |
| js_devdata = &kbdev->js_data; |
| policy_info = &js_devdata->policy.cfs; |
| |
| atomic64_set(&policy_info->least_runtime_us, KBASEP_JS_RUNTIME_EMPTY); |
| atomic64_set(&policy_info->rt_least_runtime_us, KBASEP_JS_RUNTIME_EMPTY); |
| |
| policy_info->head_runtime_us = 0; |
| |
| return 0; |
| } |
| |
| void kbasep_js_policy_term(union kbasep_js_policy *js_policy) |
| { |
| CSTD_UNUSED(js_policy); |
| } |
| |
| int kbasep_js_policy_init_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) |
| { |
| struct kbasep_js_device_data *js_devdata; |
| struct kbasep_js_policy_cfs_ctx *ctx_info; |
| struct kbasep_js_policy_cfs *policy_info; |
| int policy; |
| |
| KBASE_DEBUG_ASSERT(kbdev != NULL); |
| KBASE_DEBUG_ASSERT(kctx != NULL); |
| |
| js_devdata = &kbdev->js_data; |
| policy_info = &kbdev->js_data.policy.cfs; |
| ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs; |
| |
| KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_INIT_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx)); |
| |
| policy = current->policy; |
| if (policy == SCHED_FIFO || policy == SCHED_RR) { |
| ctx_info->process_rt_policy = true; |
| ctx_info->process_priority = (((MAX_RT_PRIO - 1) - current->rt_priority) / 5) - 20; |
| } else { |
| ctx_info->process_rt_policy = false; |
| ctx_info->process_priority = (current->static_prio - MAX_RT_PRIO) - 20; |
| } |
| |
| /* Initial runtime (relative to least-run context runtime) |
| * |
| * This uses the Policy Queue's most up-to-date head_runtime_us by using the |
| * queue mutex to issue memory barriers - also ensure future updates to |
| * head_runtime_us occur strictly after this context is initialized */ |
| mutex_lock(&js_devdata->queue_mutex); |
| |
| /* No need to hold the the runpool_irq.lock here, because we're initializing |
| * the value, and the context is definitely not being updated in the |
| * runpool at this point. The queue_mutex ensures the memory barrier. */ |
| ctx_info->runtime_us = policy_info->head_runtime_us + priority_weight(ctx_info, (u64) js_devdata->cfs_ctx_runtime_init_slices * (u64) (js_devdata->ctx_timeslice_ns / 1000u)); |
| |
| mutex_unlock(&js_devdata->queue_mutex); |
| |
| return 0; |
| } |
| |
| void kbasep_js_policy_term_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx) |
| { |
| struct kbasep_js_policy_cfs_ctx *ctx_info; |
| struct kbasep_js_policy_cfs *policy_info; |
| struct kbase_device *kbdev; |
| |
| KBASE_DEBUG_ASSERT(js_policy != NULL); |
| KBASE_DEBUG_ASSERT(kctx != NULL); |
| |
| policy_info = &js_policy->cfs; |
| ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs; |
| |
| kbdev = container_of(js_policy, struct kbase_device, js_data.policy); |
| KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_TERM_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx)); |
| |
| /* No work to do */ |
| } |
| |
| /* |
| * Job Chain Management |
| */ |
| |
| void kbasep_js_policy_log_job_result(union kbasep_js_policy *js_policy, struct kbase_jd_atom *katom, u64 time_spent_us) |
| { |
| struct kbasep_js_policy_cfs_ctx *ctx_info; |
| struct kbase_context *parent_ctx; |
| |
| KBASE_DEBUG_ASSERT(js_policy != NULL); |
| KBASE_DEBUG_ASSERT(katom != NULL); |
| CSTD_UNUSED(js_policy); |
| |
| parent_ctx = katom->kctx; |
| KBASE_DEBUG_ASSERT(parent_ctx != NULL); |
| |
| ctx_info = &parent_ctx->jctx.sched_info.runpool.policy_ctx.cfs; |
| |
| ctx_info->runtime_us += priority_weight(ctx_info, time_spent_us); |
| |
| katom->time_spent_us += time_spent_us; |
| } |
| |
| bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy *js_policy, struct kbase_context *current_ctx, struct kbase_context *new_ctx) |
| { |
| struct kbasep_js_policy_cfs_ctx *current_ctx_info; |
| struct kbasep_js_policy_cfs_ctx *new_ctx_info; |
| |
| KBASE_DEBUG_ASSERT(current_ctx != NULL); |
| KBASE_DEBUG_ASSERT(new_ctx != NULL); |
| CSTD_UNUSED(js_policy); |
| |
| current_ctx_info = ¤t_ctx->jctx.sched_info.runpool.policy_ctx.cfs; |
| new_ctx_info = &new_ctx->jctx.sched_info.runpool.policy_ctx.cfs; |
| |
| if (!current_ctx_info->process_rt_policy && new_ctx_info->process_rt_policy) |
| return true; |
| |
| if (current_ctx_info->process_rt_policy == |
| new_ctx_info->process_rt_policy) |
| return true; |
| |
| return false; |
| } |