bifrost/r38p2/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_csf.c - manifest_repos/mali-driver - Git at Google

 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2019-2022 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
  * Foundation, and any use by you of this program is subject to the terms
  * of such GNU license.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, you can access it online at
  * http://www.gnu.org/licenses/gpl-2.0.html.
  *
  */

 #include <device/mali_kbase_device_internal.h>
 #include <device/mali_kbase_device.h>

 #include <mali_kbase_hwaccess_backend.h>
 #include <mali_kbase_hwcnt_backend_csf_if_fw.h>
 #include <mali_kbase_hwcnt_watchdog_if_timer.h>
 #include <mali_kbase_ctx_sched.h>
 #include <mali_kbase_reset_gpu.h>
 #include <csf/mali_kbase_csf.h>
 #include <csf/ipa_control/mali_kbase_csf_ipa_control.h>

 #if IS_ENABLED(CONFIG_MALI_NO_MALI)
 #include <backend/gpu/mali_kbase_model_linux.h>
 #endif

 #include <mali_kbase.h>
 #include <backend/gpu/mali_kbase_irq_internal.h>
 #include <backend/gpu/mali_kbase_pm_internal.h>
 #include <backend/gpu/mali_kbase_js_internal.h>
 #include <backend/gpu/mali_kbase_clk_rate_trace_mgr.h>
 #include <csf/mali_kbase_csf_csg_debugfs.h>
 #include <mali_kbase_hwcnt_virtualizer.h>
 #include <mali_kbase_kinstr_prfcnt.h>
 #include <mali_kbase_vinstr.h>
 #include <tl/mali_kbase_timeline.h>

 /**
  * kbase_device_firmware_hwcnt_term - Terminate CSF firmware and HWC
  *
  * @kbdev: An instance of the GPU platform device, allocated from the probe
  *         method of the driver.
  *
  * When a kbase driver is removed, terminate CSF firmware and hardware counter
  * components.
  */
 static void kbase_device_firmware_hwcnt_term(struct kbase_device *kbdev)
 {
 	if (kbdev->csf.firmware_inited) {
 		kbase_kinstr_prfcnt_term(kbdev->kinstr_prfcnt_ctx);
 		kbase_vinstr_term(kbdev->vinstr_ctx);
 		kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt);
 		kbase_hwcnt_backend_csf_metadata_term(&kbdev->hwcnt_gpu_iface);
 		kbase_csf_firmware_unload_term(kbdev);
 	}
 }

 /**
  * kbase_backend_late_init - Perform any backend-specific initialization.
  * @kbdev:	Device pointer
  *
  * Return: 0 on success, or an error code on failure.
  */
 static int kbase_backend_late_init(struct kbase_device *kbdev)
 {
 	int err;

 	err = kbase_hwaccess_pm_init(kbdev);
 	if (err)
 		return err;

 	err = kbase_reset_gpu_init(kbdev);
 	if (err)
 		goto fail_reset_gpu_init;

 	err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT);
 	if (err)
 		goto fail_pm_powerup;

 	err = kbase_backend_timer_init(kbdev);
 	if (err)
 		goto fail_timer;

 #ifdef CONFIG_MALI_DEBUG
 #ifndef CONFIG_MALI_NO_MALI
 	if (kbasep_common_test_interrupt_handlers(kbdev) != 0) {
 		dev_err(kbdev->dev, "Interrupt assignment check failed.\n");
 		err = -EINVAL;
 		goto fail_interrupt_test;
 	}
 #endif /* !CONFIG_MALI_NO_MALI */
 #endif /* CONFIG_MALI_DEBUG */

 	kbase_ipa_control_init(kbdev);

 	/* Initialise the metrics subsystem, it couldn't be initialized earlier
 	 * due to dependency on kbase_ipa_control.
 	 */
 	err = kbasep_pm_metrics_init(kbdev);
 	if (err)
 		goto fail_pm_metrics_init;

 	/* Do the initialisation of devfreq.
 	 * Devfreq needs backend_timer_init() for completion of its
 	 * initialisation and it also needs to catch the first callback
 	 * occurrence of the runtime_suspend event for maintaining state
 	 * coherence with the backend power management, hence needs to be
 	 * placed before the kbase_pm_context_idle().
 	 */
 	err = kbase_backend_devfreq_init(kbdev);
 	if (err)
 		goto fail_devfreq_init;

 	/* Update gpuprops with L2_FEATURES if applicable */
 	err = kbase_gpuprops_update_l2_features(kbdev);
 	if (err)
 		goto fail_update_l2_features;

 	init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait);

 	kbase_pm_context_idle(kbdev);

 	mutex_init(&kbdev->fw_load_lock);

 	return 0;

 fail_update_l2_features:
 	kbase_backend_devfreq_term(kbdev);
 fail_devfreq_init:
 	kbasep_pm_metrics_term(kbdev);
 fail_pm_metrics_init:
 	kbase_ipa_control_term(kbdev);

 #ifdef CONFIG_MALI_DEBUG
 #ifndef CONFIG_MALI_NO_MALI
 fail_interrupt_test:
 #endif /* !CONFIG_MALI_NO_MALI */
 #endif /* CONFIG_MALI_DEBUG */

 	kbase_backend_timer_term(kbdev);
 fail_timer:
 	kbase_pm_context_idle(kbdev);
 	kbase_hwaccess_pm_halt(kbdev);
 fail_pm_powerup:
 	kbase_reset_gpu_term(kbdev);
 fail_reset_gpu_init:
 	kbase_hwaccess_pm_term(kbdev);

 	return err;
 }

 /**
  * kbase_backend_late_term - Perform any backend-specific termination.
  * @kbdev:	Device pointer
  */
 static void kbase_backend_late_term(struct kbase_device *kbdev)
 {
 	kbase_backend_devfreq_term(kbdev);
 	kbasep_pm_metrics_term(kbdev);
 	kbase_ipa_control_term(kbdev);
 	kbase_hwaccess_pm_halt(kbdev);
 	kbase_reset_gpu_term(kbdev);
 	kbase_hwaccess_pm_term(kbdev);
 }

 /**
  * kbase_csf_early_init - Early initialization for firmware & scheduler.
  * @kbdev:	Device pointer
  *
  * Return: 0 on success, error code otherwise.
  */
 static int kbase_csf_early_init(struct kbase_device *kbdev)
 {
 	int err = kbase_csf_firmware_early_init(kbdev);

 	if (err)
 		return err;

 	err = kbase_csf_scheduler_early_init(kbdev);

 	return err;
 }

 /**
  * kbase_csf_early_term() - Early termination for firmware & scheduler.
  * @kbdev:	Device pointer
  */
 static void kbase_csf_early_term(struct kbase_device *kbdev)
 {
 	kbase_csf_scheduler_early_term(kbdev);
 	kbase_csf_firmware_early_term(kbdev);
 }

 /**
  * kbase_csf_late_init - late initialization for firmware.
  * @kbdev:	Device pointer
  *
  * Return: 0 on success, error code otherwise.
  */
 static int kbase_csf_late_init(struct kbase_device *kbdev)
 {
 	int err = kbase_csf_firmware_late_init(kbdev);

 	return err;
 }

 /**
  * kbase_device_hwcnt_watchdog_if_init - Create hardware counter watchdog
  *                                       interface.
  * @kbdev:	Device pointer
  *
  * Return: 0 if successful or a negative error code on failure.
  */
 static int kbase_device_hwcnt_watchdog_if_init(struct kbase_device *kbdev)
 {
 	return kbase_hwcnt_watchdog_if_timer_create(
 		&kbdev->hwcnt_watchdog_timer);
 }

 /**
  * kbase_device_hwcnt_watchdog_if_term - Terminate hardware counter watchdog
  *                                       interface.
  * @kbdev:	Device pointer
  */
 static void kbase_device_hwcnt_watchdog_if_term(struct kbase_device *kbdev)
 {
 	kbase_hwcnt_watchdog_if_timer_destroy(&kbdev->hwcnt_watchdog_timer);
 }

 /**
  * kbase_device_hwcnt_backend_csf_if_init - Create hardware counter backend
  *                                          firmware interface.
  * @kbdev:	Device pointer
  * Return: 0 if successful or a negative error code on failure.
  */
 static int kbase_device_hwcnt_backend_csf_if_init(struct kbase_device *kbdev)
 {
 	return kbase_hwcnt_backend_csf_if_fw_create(
 		kbdev, &kbdev->hwcnt_backend_csf_if_fw);
 }

 /**
  * kbase_device_hwcnt_backend_csf_if_term - Terminate hardware counter backend
  *                                          firmware interface.
  * @kbdev:	Device pointer
  */
 static void kbase_device_hwcnt_backend_csf_if_term(struct kbase_device *kbdev)
 {
 	kbase_hwcnt_backend_csf_if_fw_destroy(&kbdev->hwcnt_backend_csf_if_fw);
 }

 /**
  * kbase_device_hwcnt_backend_csf_init - Create hardware counter backend.
  * @kbdev:	Device pointer
  *
  * Return: 0 if successful or a negative error code on failure.
  */
 static int kbase_device_hwcnt_backend_csf_init(struct kbase_device *kbdev)
 {
 	return kbase_hwcnt_backend_csf_create(
 		&kbdev->hwcnt_backend_csf_if_fw,
 		KBASE_HWCNT_BACKEND_CSF_RING_BUFFER_COUNT,
 		&kbdev->hwcnt_watchdog_timer, &kbdev->hwcnt_gpu_iface);
 }

 /**
  * kbase_device_hwcnt_backend_csf_term - Terminate hardware counter backend.
  * @kbdev:	Device pointer
  */
 static void kbase_device_hwcnt_backend_csf_term(struct kbase_device *kbdev)
 {
 	kbase_hwcnt_backend_csf_destroy(&kbdev->hwcnt_gpu_iface);
 }

 static const struct kbase_device_init dev_init[] = {
 #if IS_ENABLED(CONFIG_MALI_NO_MALI)
 	{ kbase_gpu_device_create, kbase_gpu_device_destroy, "Dummy model initialization failed" },
 #else
 	{ assign_irqs, NULL, "IRQ search failed" },
 	{ registers_map, registers_unmap, "Register map failed" },
 #endif
 	{ power_control_init, power_control_term, "Power control initialization failed" },
 	{ kbase_device_io_history_init, kbase_device_io_history_term,
 	  "Register access history initialization failed" },
 	{ kbase_device_early_init, kbase_device_early_term, "Early device initialization failed" },
 	{ kbase_device_populate_max_freq, NULL, "Populating max frequency failed" },
 	{ kbase_pm_lowest_gpu_freq_init, NULL, "Lowest freq initialization failed" },
 	{ kbase_device_misc_init, kbase_device_misc_term,
 	  "Miscellaneous device initialization failed" },
 	{ kbase_device_pcm_dev_init, kbase_device_pcm_dev_term,
 	  "Priority control manager initialization failed" },
 	{ kbase_ctx_sched_init, kbase_ctx_sched_term, "Context scheduler initialization failed" },
 	{ kbase_mem_init, kbase_mem_term, "Memory subsystem initialization failed" },
 	{ kbase_csf_protected_memory_init, kbase_csf_protected_memory_term,
 	  "Protected memory allocator initialization failed" },
 	{ kbase_device_coherency_init, NULL, "Device coherency init failed" },
 	{ kbase_protected_mode_init, kbase_protected_mode_term,
 	  "Protected mode subsystem initialization failed" },
 	{ kbase_device_list_init, kbase_device_list_term, "Device list setup failed" },
 	{ kbase_device_timeline_init, kbase_device_timeline_term,
 	  "Timeline stream initialization failed" },
 	{ kbase_clk_rate_trace_manager_init, kbase_clk_rate_trace_manager_term,
 	  "Clock rate trace manager initialization failed" },
 	{ kbase_device_hwcnt_watchdog_if_init, kbase_device_hwcnt_watchdog_if_term,
 	  "GPU hwcnt backend watchdog interface creation failed" },
 	{ kbase_device_hwcnt_backend_csf_if_init, kbase_device_hwcnt_backend_csf_if_term,
 	  "GPU hwcnt backend CSF interface creation failed" },
 	{ kbase_device_hwcnt_backend_csf_init, kbase_device_hwcnt_backend_csf_term,
 	  "GPU hwcnt backend creation failed" },
 	{ kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term,
 	  "GPU hwcnt context initialization failed" },
 	{ kbase_csf_early_init, kbase_csf_early_term, "Early CSF initialization failed" },
 	{ kbase_backend_late_init, kbase_backend_late_term, "Late backend initialization failed" },
 	{ kbase_csf_late_init, NULL, "Late CSF initialization failed" },
 	{ NULL, kbase_device_firmware_hwcnt_term, NULL },
 	{ kbase_debug_csf_fault_init, kbase_debug_csf_fault_term,
 	  "CSF fault debug initialization failed" },
 	{ kbase_device_debugfs_init, kbase_device_debugfs_term, "DebugFS initialization failed" },
 	/* Sysfs init needs to happen before registering the device with
 	 * misc_register(), otherwise it causes a race condition between
 	 * registering the device and a uevent event being generated for
 	 * userspace, causing udev rules to run which might expect certain
 	 * sysfs attributes present. As a result of the race condition
 	 * we avoid, some Mali sysfs entries may have appeared to udev
 	 * to not exist.
 	 * For more information, see
 	 * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the
 	 * paragraph that starts with "Word of warning", currently the
 	 * second-last paragraph.
 	 */
 	{ kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed" },
 	{ kbase_device_misc_register, kbase_device_misc_deregister,
 	  "Misc device registration failed" },
 	{ kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer,
 	  "GPU property population failed" },
 	{ kbase_device_late_init, kbase_device_late_term, "Late device initialization failed" },
 };

 static void kbase_device_term_partial(struct kbase_device *kbdev,
 		unsigned int i)
 {
 	while (i-- > 0) {
 		if (dev_init[i].term)
 			dev_init[i].term(kbdev);
 	}
 }

 void kbase_device_term(struct kbase_device *kbdev)
 {
 	kbdev->csf.mali_file_inode = NULL;
 	kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init));
 	kbase_mem_halt(kbdev);
 }

 int kbase_device_init(struct kbase_device *kbdev)
 {
 	int err = 0;
 	unsigned int i = 0;

 	dev_info(kbdev->dev, "Kernel DDK version %s", MALI_RELEASE_NAME);

 	kbase_device_id_init(kbdev);
 	kbase_disjoint_init(kbdev);

 	for (i = 0; i < ARRAY_SIZE(dev_init); i++) {
 		if (dev_init[i].init) {
 			err = dev_init[i].init(kbdev);
 			if (err) {
 				dev_err(kbdev->dev, "%s error = %d\n",
 					dev_init[i].err_mes, err);
 				kbase_device_term_partial(kbdev, i);
 				break;
 			}
 		}
 	}

 	return err;
 }

 /**
  * kbase_device_hwcnt_csf_deferred_init - Initialize CSF deferred HWC components
  *
  * @kbdev: An instance of the GPU platform device, allocated from the probe
  *         method of the driver.
  *
  * Hardware counter components depending on firmware are initialized after CSF
  * firmware is loaded.
  *
  * Return: 0 on success. An error code on failure.
  */
 static int kbase_device_hwcnt_csf_deferred_init(struct kbase_device *kbdev)
 {
 	int ret = 0;

 	/* For CSF GPUs, HWC metadata needs to query information from CSF
 	 * firmware, so the initialization of HWC metadata only can be called
 	 * after firmware initialized, but firmware initialization depends on
 	 * HWC backend initialization, so we need to separate HWC backend
 	 * metadata initialization from HWC backend initialization.
 	 */
 	ret = kbase_hwcnt_backend_csf_metadata_init(&kbdev->hwcnt_gpu_iface);
 	if (ret) {
 		dev_err(kbdev->dev,
 			"GPU hwcnt backend metadata creation failed");
 		return ret;
 	}

 	ret = kbase_hwcnt_virtualizer_init(
 		kbdev->hwcnt_gpu_ctx,
 		KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS,
 		&kbdev->hwcnt_gpu_virt);
 	if (ret) {
 		dev_err(kbdev->dev,
 			"GPU hwcnt virtualizer initialization failed");
 		goto virt_fail;
 	}

 	ret = kbase_vinstr_init(kbdev->hwcnt_gpu_virt, &kbdev->vinstr_ctx);
 	if (ret) {
 		dev_err(kbdev->dev,
 			"Virtual instrumentation initialization failed");
 		goto vinstr_fail;
 	}

 	ret = kbase_kinstr_prfcnt_init(kbdev->hwcnt_gpu_virt,
 				       &kbdev->kinstr_prfcnt_ctx);
 	if (ret) {
 		dev_err(kbdev->dev,
 			"Performance counter instrumentation initialization failed");
 		goto kinstr_prfcnt_fail;
 	}

 	return ret;

 kinstr_prfcnt_fail:
 	kbase_vinstr_term(kbdev->vinstr_ctx);

 vinstr_fail:
 	kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt);

 virt_fail:
 	kbase_hwcnt_backend_csf_metadata_term(&kbdev->hwcnt_gpu_iface);
 	return ret;
 }

 /**
  * kbase_csf_firmware_deferred_init - Load and initialize CSF firmware
  *
  * @kbdev: An instance of the GPU platform device, allocated from the probe
  *         method of the driver.
  *
  * Called when a device file is opened for the first time.
  * To meet Android GKI vendor guideline, firmware load is deferred at
  * the time when @ref kbase_open is called for the first time.
  *
  * Return: 0 on success. An error code on failure.
  */
 static int kbase_csf_firmware_deferred_init(struct kbase_device *kbdev)
 {
 	int err = 0;

 	lockdep_assert_held(&kbdev->fw_load_lock);

 	err = kbase_csf_firmware_load_init(kbdev);
 	if (!err) {
 		unsigned long flags;

 		spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 		kbdev->pm.backend.mcu_state = KBASE_MCU_ON;
 		kbdev->csf.firmware_inited = true;
 		spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 	} else {
 		dev_err(kbdev->dev, "Firmware initialization failed");
 	}

 	return err;
 }

 int kbase_device_firmware_init_once(struct kbase_device *kbdev)
 {
 	int ret = 0;

 	mutex_lock(&kbdev->fw_load_lock);

 	if (!kbdev->csf.firmware_inited) {
 		kbase_pm_context_active(kbdev);

 		ret = kbase_csf_firmware_deferred_init(kbdev);
 		if (ret)
 			goto out;

 		ret = kbase_device_hwcnt_csf_deferred_init(kbdev);
 		if (ret) {
 			kbase_csf_firmware_unload_term(kbdev);
 			goto out;
 		}

 		kbase_csf_debugfs_init(kbdev);
 		kbase_timeline_io_debugfs_init(kbdev);
 out:
 		kbase_pm_context_idle(kbdev);
 	}

 	mutex_unlock(&kbdev->fw_load_lock);

 	return ret;
 }
 KBASE_EXPORT_TEST_API(kbase_device_firmware_init_once);
	// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
	/*
	*
	* (C) COPYRIGHT 2019-2022 ARM Limited. All rights reserved.
	*
	* This program is free software and is provided to you under the terms of the
	* GNU General Public License version 2 as published by the Free Software
	* Foundation, and any use by you of this program is subject to the terms
	* of such GNU license.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public License
	* along with this program; if not, you can access it online at
	* http://www.gnu.org/licenses/gpl-2.0.html.
	*
	*/

	#include <device/mali_kbase_device_internal.h>
	#include <device/mali_kbase_device.h>

	#include <mali_kbase_hwaccess_backend.h>
	#include <mali_kbase_hwcnt_backend_csf_if_fw.h>
	#include <mali_kbase_hwcnt_watchdog_if_timer.h>
	#include <mali_kbase_ctx_sched.h>
	#include <mali_kbase_reset_gpu.h>
	#include <csf/mali_kbase_csf.h>
	#include <csf/ipa_control/mali_kbase_csf_ipa_control.h>

	#if IS_ENABLED(CONFIG_MALI_NO_MALI)
	#include <backend/gpu/mali_kbase_model_linux.h>
	#endif

	#include <mali_kbase.h>
	#include <backend/gpu/mali_kbase_irq_internal.h>
	#include <backend/gpu/mali_kbase_pm_internal.h>
	#include <backend/gpu/mali_kbase_js_internal.h>
	#include <backend/gpu/mali_kbase_clk_rate_trace_mgr.h>
	#include <csf/mali_kbase_csf_csg_debugfs.h>
	#include <mali_kbase_hwcnt_virtualizer.h>
	#include <mali_kbase_kinstr_prfcnt.h>
	#include <mali_kbase_vinstr.h>
	#include <tl/mali_kbase_timeline.h>

	/**
	* kbase_device_firmware_hwcnt_term - Terminate CSF firmware and HWC
	*
	* @kbdev: An instance of the GPU platform device, allocated from the probe
	* method of the driver.
	*
	* When a kbase driver is removed, terminate CSF firmware and hardware counter
	* components.
	*/
	static void kbase_device_firmware_hwcnt_term(struct kbase_device *kbdev)
	{
	if (kbdev->csf.firmware_inited) {
	kbase_kinstr_prfcnt_term(kbdev->kinstr_prfcnt_ctx);
	kbase_vinstr_term(kbdev->vinstr_ctx);
	kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt);
	kbase_hwcnt_backend_csf_metadata_term(&kbdev->hwcnt_gpu_iface);
	kbase_csf_firmware_unload_term(kbdev);
	}
	}

	/**
	* kbase_backend_late_init - Perform any backend-specific initialization.
	* @kbdev: Device pointer
	*
	* Return: 0 on success, or an error code on failure.
	*/
	static int kbase_backend_late_init(struct kbase_device *kbdev)
	{
	int err;

	err = kbase_hwaccess_pm_init(kbdev);
	if (err)
	return err;

	err = kbase_reset_gpu_init(kbdev);
	if (err)
	goto fail_reset_gpu_init;

	err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT);
	if (err)
	goto fail_pm_powerup;

	err = kbase_backend_timer_init(kbdev);
	if (err)
	goto fail_timer;

	#ifdef CONFIG_MALI_DEBUG
	#ifndef CONFIG_MALI_NO_MALI
	if (kbasep_common_test_interrupt_handlers(kbdev) != 0) {
	dev_err(kbdev->dev, "Interrupt assignment check failed.\n");
	err = -EINVAL;
	goto fail_interrupt_test;
	}
	#endif /* !CONFIG_MALI_NO_MALI */
	#endif /* CONFIG_MALI_DEBUG */

	kbase_ipa_control_init(kbdev);

	/* Initialise the metrics subsystem, it couldn't be initialized earlier
	* due to dependency on kbase_ipa_control.
	*/
	err = kbasep_pm_metrics_init(kbdev);
	if (err)
	goto fail_pm_metrics_init;

	/* Do the initialisation of devfreq.
	* Devfreq needs backend_timer_init() for completion of its
	* initialisation and it also needs to catch the first callback
	* occurrence of the runtime_suspend event for maintaining state
	* coherence with the backend power management, hence needs to be
	* placed before the kbase_pm_context_idle().
	*/
	err = kbase_backend_devfreq_init(kbdev);
	if (err)
	goto fail_devfreq_init;

	/* Update gpuprops with L2_FEATURES if applicable */
	err = kbase_gpuprops_update_l2_features(kbdev);
	if (err)
	goto fail_update_l2_features;

	init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait);

	kbase_pm_context_idle(kbdev);

	mutex_init(&kbdev->fw_load_lock);

	return 0;

	fail_update_l2_features:
	kbase_backend_devfreq_term(kbdev);
	fail_devfreq_init:
	kbasep_pm_metrics_term(kbdev);
	fail_pm_metrics_init:
	kbase_ipa_control_term(kbdev);

	#ifdef CONFIG_MALI_DEBUG
	#ifndef CONFIG_MALI_NO_MALI
	fail_interrupt_test:
	#endif /* !CONFIG_MALI_NO_MALI */
	#endif /* CONFIG_MALI_DEBUG */

	kbase_backend_timer_term(kbdev);
	fail_timer:
	kbase_pm_context_idle(kbdev);
	kbase_hwaccess_pm_halt(kbdev);
	fail_pm_powerup:
	kbase_reset_gpu_term(kbdev);
	fail_reset_gpu_init:
	kbase_hwaccess_pm_term(kbdev);

	return err;
	}

	/**
	* kbase_backend_late_term - Perform any backend-specific termination.
	* @kbdev: Device pointer
	*/
	static void kbase_backend_late_term(struct kbase_device *kbdev)
	{
	kbase_backend_devfreq_term(kbdev);
	kbasep_pm_metrics_term(kbdev);
	kbase_ipa_control_term(kbdev);
	kbase_hwaccess_pm_halt(kbdev);
	kbase_reset_gpu_term(kbdev);
	kbase_hwaccess_pm_term(kbdev);
	}

	/**
	* kbase_csf_early_init - Early initialization for firmware & scheduler.
	* @kbdev: Device pointer
	*
	* Return: 0 on success, error code otherwise.
	*/
	static int kbase_csf_early_init(struct kbase_device *kbdev)
	{
	int err = kbase_csf_firmware_early_init(kbdev);

	if (err)
	return err;

	err = kbase_csf_scheduler_early_init(kbdev);

	return err;
	}

	/**
	* kbase_csf_early_term() - Early termination for firmware & scheduler.
	* @kbdev: Device pointer
	*/
	static void kbase_csf_early_term(struct kbase_device *kbdev)
	{
	kbase_csf_scheduler_early_term(kbdev);
	kbase_csf_firmware_early_term(kbdev);
	}

	/**
	* kbase_csf_late_init - late initialization for firmware.
	* @kbdev: Device pointer
	*
	* Return: 0 on success, error code otherwise.
	*/
	static int kbase_csf_late_init(struct kbase_device *kbdev)
	{
	int err = kbase_csf_firmware_late_init(kbdev);

	return err;
	}

	/**
	* kbase_device_hwcnt_watchdog_if_init - Create hardware counter watchdog
	* interface.
	* @kbdev: Device pointer
	*
	* Return: 0 if successful or a negative error code on failure.
	*/
	static int kbase_device_hwcnt_watchdog_if_init(struct kbase_device *kbdev)
	{
	return kbase_hwcnt_watchdog_if_timer_create(
	&kbdev->hwcnt_watchdog_timer);
	}

	/**
	* kbase_device_hwcnt_watchdog_if_term - Terminate hardware counter watchdog
	* interface.
	* @kbdev: Device pointer
	*/
	static void kbase_device_hwcnt_watchdog_if_term(struct kbase_device *kbdev)
	{
	kbase_hwcnt_watchdog_if_timer_destroy(&kbdev->hwcnt_watchdog_timer);
	}

	/**
	* kbase_device_hwcnt_backend_csf_if_init - Create hardware counter backend
	* firmware interface.
	* @kbdev: Device pointer
	* Return: 0 if successful or a negative error code on failure.
	*/
	static int kbase_device_hwcnt_backend_csf_if_init(struct kbase_device *kbdev)
	{
	return kbase_hwcnt_backend_csf_if_fw_create(
	kbdev, &kbdev->hwcnt_backend_csf_if_fw);
	}

	/**
	* kbase_device_hwcnt_backend_csf_if_term - Terminate hardware counter backend
	* firmware interface.
	* @kbdev: Device pointer
	*/
	static void kbase_device_hwcnt_backend_csf_if_term(struct kbase_device *kbdev)
	{
	kbase_hwcnt_backend_csf_if_fw_destroy(&kbdev->hwcnt_backend_csf_if_fw);
	}

	/**
	* kbase_device_hwcnt_backend_csf_init - Create hardware counter backend.
	* @kbdev: Device pointer
	*
	* Return: 0 if successful or a negative error code on failure.
	*/
	static int kbase_device_hwcnt_backend_csf_init(struct kbase_device *kbdev)
	{
	return kbase_hwcnt_backend_csf_create(
	&kbdev->hwcnt_backend_csf_if_fw,
	KBASE_HWCNT_BACKEND_CSF_RING_BUFFER_COUNT,
	&kbdev->hwcnt_watchdog_timer, &kbdev->hwcnt_gpu_iface);
	}

	/**
	* kbase_device_hwcnt_backend_csf_term - Terminate hardware counter backend.
	* @kbdev: Device pointer
	*/
	static void kbase_device_hwcnt_backend_csf_term(struct kbase_device *kbdev)
	{
	kbase_hwcnt_backend_csf_destroy(&kbdev->hwcnt_gpu_iface);
	}

	static const struct kbase_device_init dev_init[] = {
	#if IS_ENABLED(CONFIG_MALI_NO_MALI)
	{ kbase_gpu_device_create, kbase_gpu_device_destroy, "Dummy model initialization failed" },
	#else
	{ assign_irqs, NULL, "IRQ search failed" },
	{ registers_map, registers_unmap, "Register map failed" },
	#endif
	{ power_control_init, power_control_term, "Power control initialization failed" },
	{ kbase_device_io_history_init, kbase_device_io_history_term,
	"Register access history initialization failed" },
	{ kbase_device_early_init, kbase_device_early_term, "Early device initialization failed" },
	{ kbase_device_populate_max_freq, NULL, "Populating max frequency failed" },
	{ kbase_pm_lowest_gpu_freq_init, NULL, "Lowest freq initialization failed" },
	{ kbase_device_misc_init, kbase_device_misc_term,
	"Miscellaneous device initialization failed" },
	{ kbase_device_pcm_dev_init, kbase_device_pcm_dev_term,
	"Priority control manager initialization failed" },
	{ kbase_ctx_sched_init, kbase_ctx_sched_term, "Context scheduler initialization failed" },
	{ kbase_mem_init, kbase_mem_term, "Memory subsystem initialization failed" },
	{ kbase_csf_protected_memory_init, kbase_csf_protected_memory_term,
	"Protected memory allocator initialization failed" },
	{ kbase_device_coherency_init, NULL, "Device coherency init failed" },
	{ kbase_protected_mode_init, kbase_protected_mode_term,
	"Protected mode subsystem initialization failed" },
	{ kbase_device_list_init, kbase_device_list_term, "Device list setup failed" },
	{ kbase_device_timeline_init, kbase_device_timeline_term,
	"Timeline stream initialization failed" },
	{ kbase_clk_rate_trace_manager_init, kbase_clk_rate_trace_manager_term,
	"Clock rate trace manager initialization failed" },
	{ kbase_device_hwcnt_watchdog_if_init, kbase_device_hwcnt_watchdog_if_term,
	"GPU hwcnt backend watchdog interface creation failed" },
	{ kbase_device_hwcnt_backend_csf_if_init, kbase_device_hwcnt_backend_csf_if_term,
	"GPU hwcnt backend CSF interface creation failed" },
	{ kbase_device_hwcnt_backend_csf_init, kbase_device_hwcnt_backend_csf_term,
	"GPU hwcnt backend creation failed" },
	{ kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term,
	"GPU hwcnt context initialization failed" },
	{ kbase_csf_early_init, kbase_csf_early_term, "Early CSF initialization failed" },
	{ kbase_backend_late_init, kbase_backend_late_term, "Late backend initialization failed" },
	{ kbase_csf_late_init, NULL, "Late CSF initialization failed" },
	{ NULL, kbase_device_firmware_hwcnt_term, NULL },
	{ kbase_debug_csf_fault_init, kbase_debug_csf_fault_term,
	"CSF fault debug initialization failed" },
	{ kbase_device_debugfs_init, kbase_device_debugfs_term, "DebugFS initialization failed" },
	/* Sysfs init needs to happen before registering the device with
	* misc_register(), otherwise it causes a race condition between
	* registering the device and a uevent event being generated for
	* userspace, causing udev rules to run which might expect certain
	* sysfs attributes present. As a result of the race condition
	* we avoid, some Mali sysfs entries may have appeared to udev
	* to not exist.
	* For more information, see
	* https://www.kernel.org/doc/Documentation/driver-model/device.txt, the
	* paragraph that starts with "Word of warning", currently the
	* second-last paragraph.
	*/
	{ kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed" },
	{ kbase_device_misc_register, kbase_device_misc_deregister,
	"Misc device registration failed" },
	{ kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer,
	"GPU property population failed" },
	{ kbase_device_late_init, kbase_device_late_term, "Late device initialization failed" },
	};

	static void kbase_device_term_partial(struct kbase_device *kbdev,
	unsigned int i)
	{
	while (i-- > 0) {
	if (dev_init[i].term)
	dev_init[i].term(kbdev);
	}
	}

	void kbase_device_term(struct kbase_device *kbdev)
	{
	kbdev->csf.mali_file_inode = NULL;
	kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init));
	kbase_mem_halt(kbdev);
	}

	int kbase_device_init(struct kbase_device *kbdev)
	{
	int err = 0;
	unsigned int i = 0;

	dev_info(kbdev->dev, "Kernel DDK version %s", MALI_RELEASE_NAME);

	kbase_device_id_init(kbdev);
	kbase_disjoint_init(kbdev);

	for (i = 0; i < ARRAY_SIZE(dev_init); i++) {
	if (dev_init[i].init) {
	err = dev_init[i].init(kbdev);
	if (err) {
	dev_err(kbdev->dev, "%s error = %d\n",
	dev_init[i].err_mes, err);
	kbase_device_term_partial(kbdev, i);
	break;
	}
	}
	}

	return err;
	}

	/**
	* kbase_device_hwcnt_csf_deferred_init - Initialize CSF deferred HWC components
	*
	* @kbdev: An instance of the GPU platform device, allocated from the probe
	* method of the driver.
	*
	* Hardware counter components depending on firmware are initialized after CSF
	* firmware is loaded.
	*
	* Return: 0 on success. An error code on failure.
	*/
	static int kbase_device_hwcnt_csf_deferred_init(struct kbase_device *kbdev)
	{
	int ret = 0;

	/* For CSF GPUs, HWC metadata needs to query information from CSF
	* firmware, so the initialization of HWC metadata only can be called
	* after firmware initialized, but firmware initialization depends on
	* HWC backend initialization, so we need to separate HWC backend
	* metadata initialization from HWC backend initialization.
	*/
	ret = kbase_hwcnt_backend_csf_metadata_init(&kbdev->hwcnt_gpu_iface);
	if (ret) {
	dev_err(kbdev->dev,
	"GPU hwcnt backend metadata creation failed");
	return ret;
	}

	ret = kbase_hwcnt_virtualizer_init(
	kbdev->hwcnt_gpu_ctx,
	KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS,
	&kbdev->hwcnt_gpu_virt);
	if (ret) {
	dev_err(kbdev->dev,
	"GPU hwcnt virtualizer initialization failed");
	goto virt_fail;
	}

	ret = kbase_vinstr_init(kbdev->hwcnt_gpu_virt, &kbdev->vinstr_ctx);
	if (ret) {
	dev_err(kbdev->dev,
	"Virtual instrumentation initialization failed");
	goto vinstr_fail;
	}

	ret = kbase_kinstr_prfcnt_init(kbdev->hwcnt_gpu_virt,
	&kbdev->kinstr_prfcnt_ctx);
	if (ret) {
	dev_err(kbdev->dev,
	"Performance counter instrumentation initialization failed");
	goto kinstr_prfcnt_fail;
	}

	return ret;

	kinstr_prfcnt_fail:
	kbase_vinstr_term(kbdev->vinstr_ctx);

	vinstr_fail:
	kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt);

	virt_fail:
	kbase_hwcnt_backend_csf_metadata_term(&kbdev->hwcnt_gpu_iface);
	return ret;
	}

	/**
	* kbase_csf_firmware_deferred_init - Load and initialize CSF firmware
	*
	* @kbdev: An instance of the GPU platform device, allocated from the probe
	* method of the driver.
	*
	* Called when a device file is opened for the first time.
	* To meet Android GKI vendor guideline, firmware load is deferred at
	* the time when @ref kbase_open is called for the first time.
	*
	* Return: 0 on success. An error code on failure.
	*/
	static int kbase_csf_firmware_deferred_init(struct kbase_device *kbdev)
	{
	int err = 0;

	lockdep_assert_held(&kbdev->fw_load_lock);

	err = kbase_csf_firmware_load_init(kbdev);
	if (!err) {
	unsigned long flags;

	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
	kbdev->pm.backend.mcu_state = KBASE_MCU_ON;
	kbdev->csf.firmware_inited = true;
	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
	} else {
	dev_err(kbdev->dev, "Firmware initialization failed");
	}

	return err;
	}

	int kbase_device_firmware_init_once(struct kbase_device *kbdev)
	{
	int ret = 0;

	mutex_lock(&kbdev->fw_load_lock);

	if (!kbdev->csf.firmware_inited) {
	kbase_pm_context_active(kbdev);

	ret = kbase_csf_firmware_deferred_init(kbdev);
	if (ret)
	goto out;

	ret = kbase_device_hwcnt_csf_deferred_init(kbdev);
	if (ret) {
	kbase_csf_firmware_unload_term(kbdev);
	goto out;
	}

	kbase_csf_debugfs_init(kbdev);
	kbase_timeline_io_debugfs_init(kbdev);
	out:
	kbase_pm_context_idle(kbdev);
	}

	mutex_unlock(&kbdev->fw_load_lock);

	return ret;
	}
	KBASE_EXPORT_TEST_API(kbase_device_firmware_init_once);