bifrost/r38p2/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_clk_rate_trace_mgr.c - manifest_repos/mali-driver - Git at Google

 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2020-2022 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
  * Foundation, and any use by you of this program is subject to the terms
  * of such GNU license.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, you can access it online at
  * http://www.gnu.org/licenses/gpl-2.0.html.
  *
  */

 /*
  * Implementation of the GPU clock rate trace manager.
  */

 #include <mali_kbase.h>
 #include <mali_kbase_config_defaults.h>
 #include <linux/clk.h>
 #include <linux/pm_opp.h>
 #include <asm/div64.h>
 #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"

 #ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY
 #include <trace/events/power_gpu_frequency.h>
 #else
 #include "mali_power_gpu_frequency_trace.h"
 #endif

 #ifndef CLK_RATE_TRACE_OPS
 #define CLK_RATE_TRACE_OPS (NULL)
 #endif

 /**
  * get_clk_rate_trace_callbacks() - Returns pointer to clk trace ops.
  * @kbdev: Pointer to kbase device, used to check if arbitration is enabled
  *         when compiled with arbiter support.
  * Return: Pointer to clk trace ops if supported or NULL.
  */
 static struct kbase_clk_rate_trace_op_conf *
 get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev)
 {
 	/* base case */
 	struct kbase_clk_rate_trace_op_conf *callbacks =
 		(struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS;
 #if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF)
 	const void *arbiter_if_node;

 	if (WARN_ON(!kbdev) || WARN_ON(!kbdev->dev))
 		return callbacks;

 	arbiter_if_node =
 		of_get_property(kbdev->dev->of_node, "arbiter_if", NULL);
 	/* Arbitration enabled, override the callback pointer.*/
 	if (arbiter_if_node)
 		callbacks = &arb_clk_rate_trace_ops;
 	else
 		dev_dbg(kbdev->dev,
 			"Arbitration supported but disabled by platform. Leaving clk rate callbacks as default.\n");

 #endif

 	return callbacks;
 }

 static int gpu_clk_rate_change_notifier(struct notifier_block *nb,
 			unsigned long event, void *data)
 {
 	struct kbase_gpu_clk_notifier_data *ndata = data;
 	struct kbase_clk_data *clk_data =
 		container_of(nb, struct kbase_clk_data, clk_rate_change_nb);
 	struct kbase_clk_rate_trace_manager *clk_rtm = clk_data->clk_rtm;
 	unsigned long flags;

 	if (WARN_ON_ONCE(clk_data->gpu_clk_handle != ndata->gpu_clk_handle))
 		return NOTIFY_BAD;

 	spin_lock_irqsave(&clk_rtm->lock, flags);
 	if (event == POST_RATE_CHANGE) {
 		if (!clk_rtm->gpu_idle &&
 		    (clk_data->clock_val != ndata->new_rate)) {
 			kbase_clk_rate_trace_manager_notify_all(
 				clk_rtm, clk_data->index, ndata->new_rate);
 		}

 		clk_data->clock_val = ndata->new_rate;
 	}
 	spin_unlock_irqrestore(&clk_rtm->lock, flags);

 	return NOTIFY_DONE;
 }

 static int gpu_clk_data_init(struct kbase_device *kbdev,
 		void *gpu_clk_handle, unsigned int index)
 {
 	struct kbase_clk_rate_trace_op_conf *callbacks;
 	struct kbase_clk_data *clk_data;
 	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
 	int ret = 0;

 	callbacks = get_clk_rate_trace_callbacks(kbdev);

 	if (WARN_ON(!callbacks) ||
 	    WARN_ON(!gpu_clk_handle) ||
 	    WARN_ON(index >= BASE_MAX_NR_CLOCKS_REGULATORS))
 		return -EINVAL;

 	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
 	if (!clk_data) {
 		dev_err(kbdev->dev, "Failed to allocate data for clock enumerated at index %u", index);
 		return -ENOMEM;
 	}

 	clk_data->index = (u8)index;
 	clk_data->gpu_clk_handle = gpu_clk_handle;
 	/* Store the initial value of clock */
 	clk_data->clock_val =
 		callbacks->get_gpu_clk_rate(kbdev, gpu_clk_handle);

 	{
 		/* At the initialization time, GPU is powered off. */
 		unsigned long flags;

 		spin_lock_irqsave(&clk_rtm->lock, flags);
 		kbase_clk_rate_trace_manager_notify_all(
 			clk_rtm, clk_data->index, 0);
 		spin_unlock_irqrestore(&clk_rtm->lock, flags);
 	}

 	clk_data->clk_rtm = clk_rtm;
 	clk_rtm->clks[index] = clk_data;

 	clk_data->clk_rate_change_nb.notifier_call =
 			gpu_clk_rate_change_notifier;

 	if (callbacks->gpu_clk_notifier_register)
 		ret = callbacks->gpu_clk_notifier_register(kbdev,
 				gpu_clk_handle, &clk_data->clk_rate_change_nb);
 	if (ret) {
 		dev_err(kbdev->dev, "Failed to register notifier for clock enumerated at index %u", index);
 		kfree(clk_data);
 	}

 	return ret;
 }

 int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev)
 {
 	struct kbase_clk_rate_trace_op_conf *callbacks;
 	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
 	unsigned int i;
 	int ret = 0;

 	callbacks = get_clk_rate_trace_callbacks(kbdev);

 	spin_lock_init(&clk_rtm->lock);
 	INIT_LIST_HEAD(&clk_rtm->listeners);

 	/* Return early if no callbacks provided for clock rate tracing */
 	if (!callbacks) {
 		WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
 		return 0;
 	}

 	clk_rtm->gpu_idle = true;

 	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
 		void *gpu_clk_handle =
 			callbacks->enumerate_gpu_clk(kbdev, i);

 		if (!gpu_clk_handle)
 			break;

 		ret = gpu_clk_data_init(kbdev, gpu_clk_handle, i);
 		if (ret)
 			goto error;
 	}

 	/* Activate clock rate trace manager if at least one GPU clock was
 	 * enumerated.
 	 */
 	if (i) {
 		WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks);
 	} else {
 		dev_info(kbdev->dev, "No clock(s) available for rate tracing");
 		WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
 	}

 	return 0;

 error:
 	while (i--) {
 		clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister(
 				kbdev, clk_rtm->clks[i]->gpu_clk_handle,
 				&clk_rtm->clks[i]->clk_rate_change_nb);
 		kfree(clk_rtm->clks[i]);
 	}

 	return ret;
 }

 void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev)
 {
 	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
 	unsigned int i;

 	WARN_ON(!list_empty(&clk_rtm->listeners));

 	if (!clk_rtm->clk_rate_trace_ops)
 		return;

 	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
 		if (!clk_rtm->clks[i])
 			break;

 		if (clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister)
 			clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister
 			(kbdev, clk_rtm->clks[i]->gpu_clk_handle,
 			&clk_rtm->clks[i]->clk_rate_change_nb);
 		kfree(clk_rtm->clks[i]);
 	}

 	WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
 }

 void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev)
 {
 	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
 	unsigned int i;
 	unsigned long flags;

 	if (!clk_rtm->clk_rate_trace_ops)
 		return;

 	spin_lock_irqsave(&clk_rtm->lock, flags);

 	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
 		struct kbase_clk_data *clk_data = clk_rtm->clks[i];

 		if (!clk_data)
 			break;

 		if (unlikely(!clk_data->clock_val))
 			continue;

 		kbase_clk_rate_trace_manager_notify_all(
 			clk_rtm, clk_data->index, clk_data->clock_val);
 	}

 	clk_rtm->gpu_idle = false;
 	spin_unlock_irqrestore(&clk_rtm->lock, flags);
 }

 void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev)
 {
 	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
 	unsigned int i;
 	unsigned long flags;

 	if (!clk_rtm->clk_rate_trace_ops)
 		return;

 	spin_lock_irqsave(&clk_rtm->lock, flags);

 	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
 		struct kbase_clk_data *clk_data = clk_rtm->clks[i];

 		if (!clk_data)
 			break;

 		if (unlikely(!clk_data->clock_val))
 			continue;

 		kbase_clk_rate_trace_manager_notify_all(
 			clk_rtm, clk_data->index, 0);
 	}

 	clk_rtm->gpu_idle = true;
 	spin_unlock_irqrestore(&clk_rtm->lock, flags);
 }

 void kbase_clk_rate_trace_manager_notify_all(
 	struct kbase_clk_rate_trace_manager *clk_rtm,
 	u32 clk_index,
 	unsigned long new_rate)
 {
 	struct kbase_clk_rate_listener *pos;
 	struct kbase_device *kbdev;

 	lockdep_assert_held(&clk_rtm->lock);

 	kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm);

 	dev_dbg(kbdev->dev, "%s - GPU clock %u rate changed to %lu, pid: %d",
 		__func__, clk_index, new_rate, current->pid);

 	/* Raise standard `power/gpu_frequency` ftrace event */
 	{
 		unsigned long new_rate_khz = new_rate;

 #if BITS_PER_LONG == 64
 		do_div(new_rate_khz, 1000);
 #elif BITS_PER_LONG == 32
 		new_rate_khz /= 1000;
 #else
 #error "unsigned long division is not supported for this architecture"
 #endif

 		trace_gpu_frequency(new_rate_khz, clk_index);
 	}

 	/* Notify the listeners. */
 	list_for_each_entry(pos, &clk_rtm->listeners, node) {
 		pos->notify(pos, clk_index, new_rate);
 	}
 }
 KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all);
	// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
	/*
	*
	* (C) COPYRIGHT 2020-2022 ARM Limited. All rights reserved.
	*
	* This program is free software and is provided to you under the terms of the
	* GNU General Public License version 2 as published by the Free Software
	* Foundation, and any use by you of this program is subject to the terms
	* of such GNU license.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public License
	* along with this program; if not, you can access it online at
	* http://www.gnu.org/licenses/gpl-2.0.html.
	*
	*/

	/*
	* Implementation of the GPU clock rate trace manager.
	*/

	#include <mali_kbase.h>
	#include <mali_kbase_config_defaults.h>
	#include <linux/clk.h>
	#include <linux/pm_opp.h>
	#include <asm/div64.h>
	#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"

	#ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY
	#include <trace/events/power_gpu_frequency.h>
	#else
	#include "mali_power_gpu_frequency_trace.h"
	#endif

	#ifndef CLK_RATE_TRACE_OPS
	#define CLK_RATE_TRACE_OPS (NULL)
	#endif

	/**
	* get_clk_rate_trace_callbacks() - Returns pointer to clk trace ops.
	* @kbdev: Pointer to kbase device, used to check if arbitration is enabled
	* when compiled with arbiter support.
	* Return: Pointer to clk trace ops if supported or NULL.
	*/
	static struct kbase_clk_rate_trace_op_conf *
	get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev)
	{
	/* base case */
	struct kbase_clk_rate_trace_op_conf *callbacks =
	(struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS;
	#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF)
	const void *arbiter_if_node;

	if (WARN_ON(!kbdev) \|\| WARN_ON(!kbdev->dev))
	return callbacks;

	arbiter_if_node =
	of_get_property(kbdev->dev->of_node, "arbiter_if", NULL);
	/* Arbitration enabled, override the callback pointer.*/
	if (arbiter_if_node)
	callbacks = &arb_clk_rate_trace_ops;
	else
	dev_dbg(kbdev->dev,
	"Arbitration supported but disabled by platform. Leaving clk rate callbacks as default.\n");

	#endif

	return callbacks;
	}

	static int gpu_clk_rate_change_notifier(struct notifier_block *nb,
	unsigned long event, void *data)
	{
	struct kbase_gpu_clk_notifier_data *ndata = data;
	struct kbase_clk_data *clk_data =
	container_of(nb, struct kbase_clk_data, clk_rate_change_nb);
	struct kbase_clk_rate_trace_manager *clk_rtm = clk_data->clk_rtm;
	unsigned long flags;

	if (WARN_ON_ONCE(clk_data->gpu_clk_handle != ndata->gpu_clk_handle))
	return NOTIFY_BAD;

	spin_lock_irqsave(&clk_rtm->lock, flags);
	if (event == POST_RATE_CHANGE) {
	if (!clk_rtm->gpu_idle &&
	(clk_data->clock_val != ndata->new_rate)) {
	kbase_clk_rate_trace_manager_notify_all(
	clk_rtm, clk_data->index, ndata->new_rate);
	}

	clk_data->clock_val = ndata->new_rate;
	}
	spin_unlock_irqrestore(&clk_rtm->lock, flags);

	return NOTIFY_DONE;
	}

	static int gpu_clk_data_init(struct kbase_device *kbdev,
	void *gpu_clk_handle, unsigned int index)
	{
	struct kbase_clk_rate_trace_op_conf *callbacks;
	struct kbase_clk_data *clk_data;
	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
	int ret = 0;

	callbacks = get_clk_rate_trace_callbacks(kbdev);

	if (WARN_ON(!callbacks) \|\|
	WARN_ON(!gpu_clk_handle) \|\|
	WARN_ON(index >= BASE_MAX_NR_CLOCKS_REGULATORS))
	return -EINVAL;

	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
	if (!clk_data) {
	dev_err(kbdev->dev, "Failed to allocate data for clock enumerated at index %u", index);
	return -ENOMEM;
	}

	clk_data->index = (u8)index;
	clk_data->gpu_clk_handle = gpu_clk_handle;
	/* Store the initial value of clock */
	clk_data->clock_val =
	callbacks->get_gpu_clk_rate(kbdev, gpu_clk_handle);

	{
	/* At the initialization time, GPU is powered off. */
	unsigned long flags;

	spin_lock_irqsave(&clk_rtm->lock, flags);
	kbase_clk_rate_trace_manager_notify_all(
	clk_rtm, clk_data->index, 0);
	spin_unlock_irqrestore(&clk_rtm->lock, flags);
	}

	clk_data->clk_rtm = clk_rtm;
	clk_rtm->clks[index] = clk_data;

	clk_data->clk_rate_change_nb.notifier_call =
	gpu_clk_rate_change_notifier;

	if (callbacks->gpu_clk_notifier_register)
	ret = callbacks->gpu_clk_notifier_register(kbdev,
	gpu_clk_handle, &clk_data->clk_rate_change_nb);
	if (ret) {
	dev_err(kbdev->dev, "Failed to register notifier for clock enumerated at index %u", index);
	kfree(clk_data);
	}

	return ret;
	}

	int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev)
	{
	struct kbase_clk_rate_trace_op_conf *callbacks;
	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
	unsigned int i;
	int ret = 0;

	callbacks = get_clk_rate_trace_callbacks(kbdev);

	spin_lock_init(&clk_rtm->lock);
	INIT_LIST_HEAD(&clk_rtm->listeners);

	/* Return early if no callbacks provided for clock rate tracing */
	if (!callbacks) {
	WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
	return 0;
	}

	clk_rtm->gpu_idle = true;

	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
	void *gpu_clk_handle =
	callbacks->enumerate_gpu_clk(kbdev, i);

	if (!gpu_clk_handle)
	break;

	ret = gpu_clk_data_init(kbdev, gpu_clk_handle, i);
	if (ret)
	goto error;
	}

	/* Activate clock rate trace manager if at least one GPU clock was
	* enumerated.
	*/
	if (i) {
	WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks);
	} else {
	dev_info(kbdev->dev, "No clock(s) available for rate tracing");
	WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
	}

	return 0;

	error:
	while (i--) {
	clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister(
	kbdev, clk_rtm->clks[i]->gpu_clk_handle,
	&clk_rtm->clks[i]->clk_rate_change_nb);
	kfree(clk_rtm->clks[i]);
	}

	return ret;
	}

	void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev)
	{
	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
	unsigned int i;

	WARN_ON(!list_empty(&clk_rtm->listeners));

	if (!clk_rtm->clk_rate_trace_ops)
	return;

	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
	if (!clk_rtm->clks[i])
	break;

	if (clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister)
	clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister
	(kbdev, clk_rtm->clks[i]->gpu_clk_handle,
	&clk_rtm->clks[i]->clk_rate_change_nb);
	kfree(clk_rtm->clks[i]);
	}

	WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
	}

	void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev)
	{
	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
	unsigned int i;
	unsigned long flags;

	if (!clk_rtm->clk_rate_trace_ops)
	return;

	spin_lock_irqsave(&clk_rtm->lock, flags);

	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
	struct kbase_clk_data *clk_data = clk_rtm->clks[i];

	if (!clk_data)
	break;

	if (unlikely(!clk_data->clock_val))
	continue;

	kbase_clk_rate_trace_manager_notify_all(
	clk_rtm, clk_data->index, clk_data->clock_val);
	}

	clk_rtm->gpu_idle = false;
	spin_unlock_irqrestore(&clk_rtm->lock, flags);
	}

	void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev)
	{
	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
	unsigned int i;
	unsigned long flags;

	if (!clk_rtm->clk_rate_trace_ops)
	return;

	spin_lock_irqsave(&clk_rtm->lock, flags);

	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
	struct kbase_clk_data *clk_data = clk_rtm->clks[i];

	if (!clk_data)
	break;

	if (unlikely(!clk_data->clock_val))
	continue;

	kbase_clk_rate_trace_manager_notify_all(
	clk_rtm, clk_data->index, 0);
	}

	clk_rtm->gpu_idle = true;
	spin_unlock_irqrestore(&clk_rtm->lock, flags);
	}

	void kbase_clk_rate_trace_manager_notify_all(
	struct kbase_clk_rate_trace_manager *clk_rtm,
	u32 clk_index,
	unsigned long new_rate)
	{
	struct kbase_clk_rate_listener *pos;
	struct kbase_device *kbdev;

	lockdep_assert_held(&clk_rtm->lock);

	kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm);

	dev_dbg(kbdev->dev, "%s - GPU clock %u rate changed to %lu, pid: %d",
	__func__, clk_index, new_rate, current->pid);

	/* Raise standard `power/gpu_frequency` ftrace event */
	{
	unsigned long new_rate_khz = new_rate;

	#if BITS_PER_LONG == 64
	do_div(new_rate_khz, 1000);
	#elif BITS_PER_LONG == 32
	new_rate_khz /= 1000;
	#else
	#error "unsigned long division is not supported for this architecture"
	#endif

	trace_gpu_frequency(new_rate_khz, clk_index);
	}

	/* Notify the listeners. */
	list_for_each_entry(pos, &clk_rtm->listeners, node) {
	pos->notify(pos, clk_index, new_rate);
	}
	}
	KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all);