bifrost/r12p0/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c

/*
 *
 * (C) COPYRIGHT 2010-2018 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU licence.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
 *
 * SPDX-License-Identifier: GPL-2.0
 *
 */

/*
 * Power policy API implementations
 */

#include <mali_kbase.h>
#include <mali_midg_regmap.h>
#include <mali_kbase_pm.h>
#include <mali_kbase_config_defaults.h>
#include <backend/gpu/mali_kbase_pm_internal.h>

static const struct kbase_pm_policy *const policy_list[] = {
#ifdef CONFIG_MALI_NO_MALI
	&kbase_pm_always_on_policy_ops,
	&kbase_pm_demand_policy_ops,
	&kbase_pm_coarse_demand_policy_ops,
#if !MALI_CUSTOMER_RELEASE
	&kbase_pm_demand_always_powered_policy_ops,
	&kbase_pm_fast_start_policy_ops,
#endif
#else				/* CONFIG_MALI_NO_MALI */
#if !PLATFORM_POWER_DOWN_ONLY
	&kbase_pm_demand_policy_ops,
#endif /* !PLATFORM_POWER_DOWN_ONLY */
	&kbase_pm_coarse_demand_policy_ops,
	&kbase_pm_always_on_policy_ops,
#if !MALI_CUSTOMER_RELEASE
#if !PLATFORM_POWER_DOWN_ONLY
	&kbase_pm_demand_always_powered_policy_ops,
	&kbase_pm_fast_start_policy_ops,
#endif /* !PLATFORM_POWER_DOWN_ONLY */
#endif
#endif /* CONFIG_MALI_NO_MALI */
};

/* The number of policies available in the system.
 * This is derived from the number of functions listed in policy_get_functions.
 */
#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))


/* Function IDs for looking up Timeline Trace codes in
 * kbase_pm_change_state_trace_code */
enum kbase_pm_func_id {
	KBASE_PM_FUNC_ID_REQUEST_CORES_START,
	KBASE_PM_FUNC_ID_REQUEST_CORES_END,
	KBASE_PM_FUNC_ID_RELEASE_CORES_START,
	KBASE_PM_FUNC_ID_RELEASE_CORES_END,
	/* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither
	 * expect to hit it nor tend to hit it very much anyway. We can detect
	 * whether we need more instrumentation by a difference between
	 * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */

	/* Must be the last */
	KBASE_PM_FUNC_ID_COUNT
};


/* State changes during request/unrequest/release-ing cores */
enum {
	KBASE_PM_CHANGE_STATE_SHADER = (1u << 0),
	KBASE_PM_CHANGE_STATE_TILER  = (1u << 1),

	/* These two must be last */
	KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER |
						KBASE_PM_CHANGE_STATE_SHADER),
	KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1
};
typedef u32 kbase_pm_change_state;


#ifdef CONFIG_MALI_TRACE_TIMELINE
/* Timeline Trace code lookups for each function */
static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT]
					[KBASE_PM_CHANGE_STATE_COUNT] = {
	/* kbase_pm_request_cores */
	[KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0,
	[KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
		SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START,
	[KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
		SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START,
	[KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
						KBASE_PM_CHANGE_STATE_TILER] =
		SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START,

	[KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0,
	[KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
		SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END,
	[KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
		SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END,
	[KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
						KBASE_PM_CHANGE_STATE_TILER] =
		SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END,

	/* kbase_pm_release_cores */
	[KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0,
	[KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
		SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START,
	[KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
		SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START,
	[KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
						KBASE_PM_CHANGE_STATE_TILER] =
		SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START,

	[KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0,
	[KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
		SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END,
	[KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
		SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END,
	[KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
						KBASE_PM_CHANGE_STATE_TILER] =
		SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END
};

static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
		enum kbase_pm_func_id func_id,
		kbase_pm_change_state state)
{
	int trace_code;

	KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT);
	KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) ==
									state);

	trace_code = kbase_pm_change_state_trace_code[func_id][state];
	KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code);
}

#else /* CONFIG_MALI_TRACE_TIMELINE */
static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
		enum kbase_pm_func_id func_id, kbase_pm_change_state state)
{
}

#endif /* CONFIG_MALI_TRACE_TIMELINE */

/**
 * kbasep_pm_do_poweroff_cores - Process a poweroff request and power down any
 *                               requested shader cores
 * @kbdev: Device pointer
 */
static void kbasep_pm_do_poweroff_cores(struct kbase_device *kbdev)
{
	u64 prev_shader_state = kbdev->pm.backend.desired_shader_state;
	u64 prev_tiler_state = kbdev->pm.backend.desired_tiler_state;

	lockdep_assert_held(&kbdev->hwaccess_lock);

	kbdev->pm.backend.desired_shader_state &=
			~kbdev->pm.backend.shader_poweroff_pending;
	kbdev->pm.backend.desired_tiler_state &=
			~kbdev->pm.backend.tiler_poweroff_pending;

	kbdev->pm.backend.shader_poweroff_pending = 0;
	kbdev->pm.backend.tiler_poweroff_pending = 0;

	if (prev_shader_state != kbdev->pm.backend.desired_shader_state ||
			prev_tiler_state !=
				kbdev->pm.backend.desired_tiler_state ||
			kbdev->pm.backend.ca_in_transition) {
		bool cores_are_available;

		KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
			SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START);
		cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
		KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
			SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END);

		/* Don't need 'cores_are_available',
		 * because we don't return anything */
		CSTD_UNUSED(cores_are_available);
	}
}

static enum hrtimer_restart
kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer)
{
	struct kbase_device *kbdev;
	unsigned long flags;

	kbdev = container_of(timer, struct kbase_device,
						pm.backend.gpu_poweroff_timer);

	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);

	/* It is safe for this call to do nothing if the work item is already
	 * queued. The worker function will read the must up-to-date state of
	 * kbdev->pm.backend.gpu_poweroff_pending under lock.
	 *
	 * If a state change occurs while the worker function is processing,
	 * this call will succeed as a work item can be requeued once it has
	 * started processing.
	 */
	if (kbdev->pm.backend.gpu_poweroff_pending)
		queue_work(kbdev->pm.backend.gpu_poweroff_wq,
					&kbdev->pm.backend.gpu_poweroff_work);

	if (kbdev->pm.backend.shader_poweroff_pending ||
			kbdev->pm.backend.tiler_poweroff_pending) {
		kbdev->pm.backend.shader_poweroff_pending_time--;

		KBASE_DEBUG_ASSERT(
				kbdev->pm.backend.shader_poweroff_pending_time
									>= 0);

		if (!kbdev->pm.backend.shader_poweroff_pending_time)
			kbasep_pm_do_poweroff_cores(kbdev);
	}

	if (kbdev->pm.backend.poweroff_timer_needed) {
		spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);

		hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time);

		return HRTIMER_RESTART;
	}

	kbdev->pm.backend.poweroff_timer_running = false;
	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);

	return HRTIMER_NORESTART;
}

static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data)
{
	unsigned long flags;
	struct kbase_device *kbdev;
	bool do_poweroff = false;

	kbdev = container_of(data, struct kbase_device,
						pm.backend.gpu_poweroff_work);

	mutex_lock(&kbdev->pm.lock);

	if (kbdev->pm.backend.gpu_poweroff_pending == 0) {
		mutex_unlock(&kbdev->pm.lock);
		return;
	}

	kbdev->pm.backend.gpu_poweroff_pending--;

	if (kbdev->pm.backend.gpu_poweroff_pending > 0) {
		mutex_unlock(&kbdev->pm.lock);
		return;
	}

	KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0);

	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);

	/* Only power off the GPU if a request is still pending */
	if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev))
		do_poweroff = true;

	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);

	if (do_poweroff) {
		kbdev->pm.backend.poweroff_timer_needed = false;
		hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
		kbdev->pm.backend.poweroff_timer_running = false;

		/* Power off the GPU */
		kbase_pm_do_poweroff(kbdev, false);
	}

	mutex_unlock(&kbdev->pm.lock);
}

int kbase_pm_policy_init(struct kbase_device *kbdev)
{
	struct workqueue_struct *wq;

	wq = alloc_workqueue("kbase_pm_do_poweroff",
			WQ_HIGHPRI | WQ_UNBOUND, 1);
	if (!wq)
		return -ENOMEM;

	kbdev->pm.backend.gpu_poweroff_wq = wq;
	INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work,
			kbasep_pm_do_gpu_poweroff_wq);
	hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer,
			CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	kbdev->pm.backend.gpu_poweroff_timer.function =
			kbasep_pm_do_gpu_poweroff_callback;
	kbdev->pm.backend.pm_current_policy = policy_list[0];
	kbdev->pm.backend.pm_current_policy->init(kbdev);
	kbdev->pm.gpu_poweroff_time =
			HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS);
	kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
	kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU;

	return 0;
}

void kbase_pm_policy_term(struct kbase_device *kbdev)
{
	kbdev->pm.backend.pm_current_policy->term(kbdev);
	destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq);
}

void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev)
{
	unsigned long flags;

	lockdep_assert_held(&kbdev->pm.lock);

	kbdev->pm.backend.poweroff_timer_needed = false;
	hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
	kbdev->pm.backend.poweroff_timer_running = false;

	/* If wq is already running but is held off by pm.lock, make sure it has
	 * no effect */
	kbdev->pm.backend.gpu_poweroff_pending = 0;

	kbdev->pm.backend.shader_poweroff_pending = 0;
	kbdev->pm.backend.tiler_poweroff_pending = 0;
	kbdev->pm.backend.shader_poweroff_pending_time = 0;

	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}

void kbase_pm_update_active(struct kbase_device *kbdev)
{
	struct kbase_pm_device_data *pm = &kbdev->pm;
	struct kbase_pm_backend_data *backend = &pm->backend;
	unsigned long flags;
	bool active;

	lockdep_assert_held(&pm->lock);

	/* pm_current_policy will never be NULL while pm.lock is held */
	KBASE_DEBUG_ASSERT(backend->pm_current_policy);

	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);

	active = backend->pm_current_policy->get_core_active(kbdev);

	if (active) {
		if (backend->gpu_poweroff_pending) {
			/* Cancel any pending power off request */
			backend->gpu_poweroff_pending = 0;

			/* If a request was pending then the GPU was still
			 * powered, so no need to continue */
			if (!kbdev->poweroff_pending) {
				spin_unlock_irqrestore(&kbdev->hwaccess_lock,
						flags);
				return;
			}
		}

		if (!backend->poweroff_timer_running && !backend->gpu_powered &&
				(pm->poweroff_gpu_ticks ||
				pm->poweroff_shader_ticks)) {
			backend->poweroff_timer_needed = true;
			backend->poweroff_timer_running = true;
			hrtimer_start(&backend->gpu_poweroff_timer,
					pm->gpu_poweroff_time,
					HRTIMER_MODE_REL);
		}

		/* Power on the GPU and any cores requested by the policy */
		if (pm->backend.poweroff_wait_in_progress) {
			KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
			pm->backend.poweron_required = true;
			spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
		} else {
			spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
			kbase_pm_do_poweron(kbdev, false);
		}
	} else {
		/* It is an error for the power policy to power off the GPU
		 * when there are contexts active */
		KBASE_DEBUG_ASSERT(pm->active_count == 0);

		if (backend->shader_poweroff_pending ||
				backend->tiler_poweroff_pending) {
			backend->shader_poweroff_pending = 0;
			backend->tiler_poweroff_pending = 0;
			backend->shader_poweroff_pending_time = 0;
		}

		/* Request power off */
		if (pm->backend.gpu_powered) {
			if (pm->poweroff_gpu_ticks) {
				backend->gpu_poweroff_pending =
						pm->poweroff_gpu_ticks;
				backend->poweroff_timer_needed = true;
				if (!backend->poweroff_timer_running) {
					/* Start timer if not running (eg if
					 * power policy has been changed from
					 * always_on to something else). This
					 * will ensure the GPU is actually
					 * powered off */
					backend->poweroff_timer_running
							= true;
					hrtimer_start(
						&backend->gpu_poweroff_timer,
						pm->gpu_poweroff_time,
						HRTIMER_MODE_REL);
				}
				spin_unlock_irqrestore(&kbdev->hwaccess_lock,
						flags);
			} else {
				spin_unlock_irqrestore(&kbdev->hwaccess_lock,
						flags);

				/* Power off the GPU immediately */
				kbase_pm_do_poweroff(kbdev, false);
			}
		} else {
			spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
		}
	}
}

void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev)
{
	u64 desired_bitmap;
	u64 desired_tiler_bitmap;
	bool cores_are_available;
	bool do_poweroff = false;

	lockdep_assert_held(&kbdev->hwaccess_lock);

	if (kbdev->pm.backend.pm_current_policy == NULL)
		return;
	if (kbdev->pm.backend.poweroff_wait_in_progress)
		return;

	if (kbdev->protected_mode_transition &&	!kbdev->shader_needed_bitmap &&
			!kbdev->shader_inuse_bitmap && !kbdev->tiler_needed_cnt
			&& !kbdev->tiler_inuse_cnt) {
		/* We are trying to change in/out of protected mode - force all
		 * cores off so that the L2 powers down */
		desired_bitmap = 0;
		desired_tiler_bitmap = 0;
	} else {
		desired_bitmap =
		kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev);
		desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev);

		if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
			desired_tiler_bitmap = 1;
		else
			desired_tiler_bitmap = 0;

		if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) {
			/* Unless XAFFINITY is supported, enable core 0 if tiler
			 * required, regardless of core availability */
			if (kbdev->tiler_needed_cnt > 0 ||
					kbdev->tiler_inuse_cnt > 0)
				desired_bitmap |= 1;
		}
	}

	if (kbdev->pm.backend.desired_shader_state != desired_bitmap)
		KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u,
							(u32)desired_bitmap);
	/* Are any cores being powered on? */
	if (~kbdev->pm.backend.desired_shader_state & desired_bitmap ||
	    ~kbdev->pm.backend.desired_tiler_state & desired_tiler_bitmap ||
	    kbdev->pm.backend.ca_in_transition) {
		/* Check if we are powering off any cores before updating shader
		 * state */
		if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap ||
				kbdev->pm.backend.desired_tiler_state &
				~desired_tiler_bitmap) {
			/* Start timer to power off cores */
			kbdev->pm.backend.shader_poweroff_pending |=
				(kbdev->pm.backend.desired_shader_state &
							~desired_bitmap);
			kbdev->pm.backend.tiler_poweroff_pending |=
				(kbdev->pm.backend.desired_tiler_state &
							~desired_tiler_bitmap);

			if (kbdev->pm.poweroff_shader_ticks &&
					!kbdev->protected_mode_transition)
				kbdev->pm.backend.shader_poweroff_pending_time =
						kbdev->pm.poweroff_shader_ticks;
			else
				do_poweroff = true;
		}

		kbdev->pm.backend.desired_shader_state = desired_bitmap;
		kbdev->pm.backend.desired_tiler_state = desired_tiler_bitmap;

		/* If any cores are being powered on, transition immediately */
		cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
	} else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap ||
				kbdev->pm.backend.desired_tiler_state &
				~desired_tiler_bitmap) {
		/* Start timer to power off cores */
		kbdev->pm.backend.shader_poweroff_pending |=
				(kbdev->pm.backend.desired_shader_state &
							~desired_bitmap);
		kbdev->pm.backend.tiler_poweroff_pending |=
				(kbdev->pm.backend.desired_tiler_state &
							~desired_tiler_bitmap);
		if (kbdev->pm.poweroff_shader_ticks &&
				!kbdev->protected_mode_transition)
			kbdev->pm.backend.shader_poweroff_pending_time =
					kbdev->pm.poweroff_shader_ticks;
		else
			kbasep_pm_do_poweroff_cores(kbdev);
	} else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 &&
			desired_tiler_bitmap != 0 &&
			kbdev->pm.backend.poweroff_timer_needed) {
		/* If power policy is keeping cores on despite there being no
		 * active contexts then disable poweroff timer as it isn't
		 * required.
		 * Only reset poweroff_timer_needed if we're not in the middle
		 * of the power off callback */
		kbdev->pm.backend.poweroff_timer_needed = false;
	}

	/* Ensure timer does not power off wanted cores and make sure to power
	 * off unwanted cores */
	if (kbdev->pm.backend.shader_poweroff_pending ||
			kbdev->pm.backend.tiler_poweroff_pending) {
		kbdev->pm.backend.shader_poweroff_pending &=
				~(kbdev->pm.backend.desired_shader_state &
								desired_bitmap);
		kbdev->pm.backend.tiler_poweroff_pending &=
				~(kbdev->pm.backend.desired_tiler_state &
				desired_tiler_bitmap);

		if (!kbdev->pm.backend.shader_poweroff_pending &&
				!kbdev->pm.backend.tiler_poweroff_pending)
			kbdev->pm.backend.shader_poweroff_pending_time = 0;
	}

	/* Shader poweroff is deferred to the end of the function, to eliminate
	 * issues caused by the core availability policy recursing into this
	 * function */
	if (do_poweroff)
		kbasep_pm_do_poweroff_cores(kbdev);

	/* Don't need 'cores_are_available', because we don't return anything */
	CSTD_UNUSED(cores_are_available);
}

void kbase_pm_update_cores_state(struct kbase_device *kbdev)
{
	unsigned long flags;

	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);

	kbase_pm_update_cores_state_nolock(kbdev);

	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}

int kbase_pm_list_policies(const struct kbase_pm_policy * const **list)
{
	if (!list)
		return POLICY_COUNT;

	*list = policy_list;

	return POLICY_COUNT;
}

KBASE_EXPORT_TEST_API(kbase_pm_list_policies);

const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev)
{
	KBASE_DEBUG_ASSERT(kbdev != NULL);

	return kbdev->pm.backend.pm_current_policy;
}

KBASE_EXPORT_TEST_API(kbase_pm_get_policy);

void kbase_pm_set_policy(struct kbase_device *kbdev,
				const struct kbase_pm_policy *new_policy)
{
	struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
	const struct kbase_pm_policy *old_policy;
	unsigned long flags;

	KBASE_DEBUG_ASSERT(kbdev != NULL);
	KBASE_DEBUG_ASSERT(new_policy != NULL);

	KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);

	/* During a policy change we pretend the GPU is active */
	/* A suspend won't happen here, because we're in a syscall from a
	 * userspace thread */
	kbase_pm_context_active(kbdev);

	mutex_lock(&js_devdata->runpool_mutex);
	mutex_lock(&kbdev->pm.lock);

	/* Remove the policy to prevent IRQ handlers from working on it */
	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
	old_policy = kbdev->pm.backend.pm_current_policy;
	kbdev->pm.backend.pm_current_policy = NULL;
	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);

	KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u,
								old_policy->id);
	if (old_policy->term)
		old_policy->term(kbdev);

	KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u,
								new_policy->id);
	if (new_policy->init)
		new_policy->init(kbdev);

	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
	kbdev->pm.backend.pm_current_policy = new_policy;
	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);

	/* If any core power state changes were previously attempted, but
	 * couldn't be made because the policy was changing (current_policy was
	 * NULL), then re-try them here. */
	kbase_pm_update_active(kbdev);
	kbase_pm_update_cores_state(kbdev);

	mutex_unlock(&kbdev->pm.lock);
	mutex_unlock(&js_devdata->runpool_mutex);

	/* Now the policy change is finished, we release our fake context active
	 * reference */
	kbase_pm_context_idle(kbdev);
}

KBASE_EXPORT_TEST_API(kbase_pm_set_policy);

/* Check whether a state change has finished, and trace it as completed */
static void
kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev)
{
	if ((kbdev->shader_available_bitmap &
					kbdev->pm.backend.desired_shader_state)
				== kbdev->pm.backend.desired_shader_state &&
		(kbdev->tiler_available_bitmap &
					kbdev->pm.backend.desired_tiler_state)
				== kbdev->pm.backend.desired_tiler_state)
		kbase_timeline_pm_check_handle_event(kbdev,
				KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
}

void kbase_pm_request_cores(struct kbase_device *kbdev,
				bool tiler_required, u64 shader_cores)
{
	u64 cores;

	kbase_pm_change_state change_gpu_state = 0u;

	KBASE_DEBUG_ASSERT(kbdev != NULL);

	lockdep_assert_held(&kbdev->hwaccess_lock);

	cores = shader_cores;
	while (cores) {
		int bitnum = fls64(cores) - 1;
		u64 bit = 1ULL << bitnum;

		/* It should be almost impossible for this to overflow. It would
		 * require 2^32 atoms to request a particular core, which would
		 * require 2^24 contexts to submit. This would require an amount
		 * of memory that is impossible on a 32-bit system and extremely
		 * unlikely on a 64-bit system. */
		int cnt = ++kbdev->shader_needed_cnt[bitnum];

		if (1 == cnt) {
			kbdev->shader_needed_bitmap |= bit;
			change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
		}

		cores &= ~bit;
	}

	if (tiler_required) {
		int cnt = ++kbdev->tiler_needed_cnt;

		if (1 == cnt)
			change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;

		KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0);
	}

	if (change_gpu_state) {
		KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL,
				NULL, 0u, (u32) kbdev->shader_needed_bitmap);

		kbase_timeline_pm_cores_func(kbdev,
					KBASE_PM_FUNC_ID_REQUEST_CORES_START,
							change_gpu_state);
		kbase_pm_update_cores_state_nolock(kbdev);
		kbase_timeline_pm_cores_func(kbdev,
					KBASE_PM_FUNC_ID_REQUEST_CORES_END,
							change_gpu_state);
	}
}

KBASE_EXPORT_TEST_API(kbase_pm_request_cores);

void kbase_pm_unrequest_cores(struct kbase_device *kbdev,
				bool tiler_required, u64 shader_cores)
{
	kbase_pm_change_state change_gpu_state = 0u;

	KBASE_DEBUG_ASSERT(kbdev != NULL);

	lockdep_assert_held(&kbdev->hwaccess_lock);

	while (shader_cores) {
		int bitnum = fls64(shader_cores) - 1;
		u64 bit = 1ULL << bitnum;
		int cnt;

		KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);

		cnt = --kbdev->shader_needed_cnt[bitnum];

		if (0 == cnt) {
			kbdev->shader_needed_bitmap &= ~bit;

			change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
		}

		shader_cores &= ~bit;
	}

	if (tiler_required) {
		int cnt;

		KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);

		cnt = --kbdev->tiler_needed_cnt;

		if (0 == cnt)
			change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
	}

	if (change_gpu_state) {
		KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL,
				NULL, 0u, (u32) kbdev->shader_needed_bitmap);

		kbase_pm_update_cores_state_nolock(kbdev);

		/* Trace that any state change effectively completes immediately
		 * - no-one will wait on the state change */
		kbase_pm_trace_check_and_finish_state_change(kbdev);
	}
}

KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores);

enum kbase_pm_cores_ready
kbase_pm_register_inuse_cores(struct kbase_device *kbdev,
				bool tiler_required, u64 shader_cores)
{
	u64 prev_shader_needed;	/* Just for tracing */
	u64 prev_shader_inuse;	/* Just for tracing */

	lockdep_assert_held(&kbdev->hwaccess_lock);

	prev_shader_needed = kbdev->shader_needed_bitmap;
	prev_shader_inuse = kbdev->shader_inuse_bitmap;

	/* If desired_shader_state does not contain the requested cores, then
	 * power management is not attempting to powering those cores (most
	 * likely due to core availability policy) and a new job affinity must
	 * be chosen */
	if ((kbdev->pm.backend.desired_shader_state & shader_cores) !=
							shader_cores) {
		return (kbdev->pm.backend.poweroff_wait_in_progress ||
				kbdev->pm.backend.pm_current_policy == NULL) ?
				KBASE_CORES_NOT_READY : KBASE_NEW_AFFINITY;
	}

	if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores ||
	    (tiler_required && !kbdev->tiler_available_bitmap)) {
		/* Trace ongoing core transition */
		kbase_timeline_pm_l2_transition_start(kbdev);
		return KBASE_CORES_NOT_READY;
	}

	/* If we started to trace a state change, then trace it has being
	 * finished by now, at the very latest */
	kbase_pm_trace_check_and_finish_state_change(kbdev);
	/* Trace core transition done */
	kbase_timeline_pm_l2_transition_done(kbdev);

	while (shader_cores) {
		int bitnum = fls64(shader_cores) - 1;
		u64 bit = 1ULL << bitnum;
		int cnt;

		KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);

		cnt = --kbdev->shader_needed_cnt[bitnum];

		if (0 == cnt)
			kbdev->shader_needed_bitmap &= ~bit;

		/* shader_inuse_cnt should not overflow because there can only
		 * be a very limited number of jobs on the h/w at one time */

		kbdev->shader_inuse_cnt[bitnum]++;
		kbdev->shader_inuse_bitmap |= bit;

		shader_cores &= ~bit;
	}

	if (tiler_required) {
		KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);

		--kbdev->tiler_needed_cnt;

		kbdev->tiler_inuse_cnt++;

		KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0);
	}

	if (prev_shader_needed != kbdev->shader_needed_bitmap)
		KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL,
				NULL, 0u, (u32) kbdev->shader_needed_bitmap);

	if (prev_shader_inuse != kbdev->shader_inuse_bitmap)
		KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL,
				NULL, 0u, (u32) kbdev->shader_inuse_bitmap);

	return KBASE_CORES_READY;
}

KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores);

void kbase_pm_release_cores(struct kbase_device *kbdev,
				bool tiler_required, u64 shader_cores)
{
	kbase_pm_change_state change_gpu_state = 0u;

	KBASE_DEBUG_ASSERT(kbdev != NULL);

	lockdep_assert_held(&kbdev->hwaccess_lock);

	while (shader_cores) {
		int bitnum = fls64(shader_cores) - 1;
		u64 bit = 1ULL << bitnum;
		int cnt;

		KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0);

		cnt = --kbdev->shader_inuse_cnt[bitnum];

		if (0 == cnt) {
			kbdev->shader_inuse_bitmap &= ~bit;
			change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
		}

		shader_cores &= ~bit;
	}

	if (tiler_required) {
		int cnt;

		KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0);

		cnt = --kbdev->tiler_inuse_cnt;

		if (0 == cnt)
			change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
	}

	if (change_gpu_state) {
		KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL,
				NULL, 0u, (u32) kbdev->shader_inuse_bitmap);

		kbase_timeline_pm_cores_func(kbdev,
					KBASE_PM_FUNC_ID_RELEASE_CORES_START,
							change_gpu_state);
		kbase_pm_update_cores_state_nolock(kbdev);
		kbase_timeline_pm_cores_func(kbdev,
					KBASE_PM_FUNC_ID_RELEASE_CORES_END,
							change_gpu_state);

		/* Trace that any state change completed immediately */
		kbase_pm_trace_check_and_finish_state_change(kbdev);
	}
}

KBASE_EXPORT_TEST_API(kbase_pm_release_cores);

void kbase_pm_request_cores_sync(struct kbase_device *kbdev,
					bool tiler_required,
					u64 shader_cores)
{
	unsigned long flags;

	kbase_pm_wait_for_poweroff_complete(kbdev);

	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
	kbase_pm_request_cores(kbdev, tiler_required, shader_cores);
	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);

	kbase_pm_check_transitions_sync(kbdev);
}

KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync);

void kbase_pm_request_l2_caches(struct kbase_device *kbdev)
{
	unsigned long flags;

	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);

	kbdev->l2_users_count++;

	KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0);

	/* Check for the required L2 transitions.
	 * Caller would block here for the L2 caches of all core groups to be
	 * powered on, so need to inform the Hw to power up all the L2 caches.
	 * Can't rely on the l2_users_count value being non-zero previously to
	 * avoid checking for the transition, as the count could be non-zero
	 * even if not all the instances of L2 cache are powered up since
	 * currently the power status of L2 is not tracked separately for each
	 * core group. Also if the GPU is reset while the L2 is on, L2 will be
	 * off but the count will be non-zero.
	 */
	kbase_pm_check_transitions_nolock(kbdev);

	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
	wait_event(kbdev->pm.backend.l2_powered_wait,
					kbdev->pm.backend.l2_powered == 1);

	/* Trace that any state change completed immediately */
	kbase_pm_trace_check_and_finish_state_change(kbdev);
}

KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches);

void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev)
{
	lockdep_assert_held(&kbdev->hwaccess_lock);

	kbdev->l2_users_count++;
}

KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on);

void kbase_pm_release_l2_caches(struct kbase_device *kbdev)
{
	lockdep_assert_held(&kbdev->hwaccess_lock);

	KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0);

	--kbdev->l2_users_count;

	if (!kbdev->l2_users_count) {
		kbase_pm_check_transitions_nolock(kbdev);
		/* Trace that any state change completed immediately */
		kbase_pm_trace_check_and_finish_state_change(kbdev);
	}
}

KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches);