bifrost/r32p1/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_hw_direct.c - manifest_repos/mali-driver - Git at Google

 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
  * Foundation, and any use by you of this program is subject to the terms
  * of such GNU license.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, you can access it online at
  * http://www.gnu.org/licenses/gpl-2.0.html.
  *
  */

 #include <linux/bitops.h>
 #include <mali_kbase.h>
 #include <mali_kbase_mem.h>
 #include <mmu/mali_kbase_mmu_hw.h>
 #include <tl/mali_kbase_tracepoints.h>
 #include <device/mali_kbase_device.h>

 /**
  * lock_region() - Generate lockaddr to lock memory region in MMU
  * @pfn:       Starting page frame number of the region to lock
  * @num_pages: Number of pages to lock. It must be greater than 0.
  * @lockaddr:  Address and size of memory region to lock
  *
  * The lockaddr value is a combination of the starting address and
  * the size of the region that encompasses all the memory pages to lock.
  *
  * The size is expressed as a logarithm: it is represented in a way
  * that is compatible with the HW specification and it also determines
  * how many of the lowest bits of the address are cleared.
  *
  * Return: 0 if success, or an error code on failure.
  */
 static int lock_region(u64 pfn, u32 num_pages, u64 *lockaddr)
 {
 	const u64 lockaddr_base = pfn << PAGE_SHIFT;
 	u64 lockaddr_size_log2, region_frame_number_start,
 		region_frame_number_end;

 	if (num_pages == 0)
 		return -EINVAL;

 	/* The size is expressed as a logarithm and should take into account
 	 * the possibility that some pages might spill into the next region.
 	 */
 	lockaddr_size_log2 = fls(num_pages) + PAGE_SHIFT - 1;

 	/* Round up if the number of pages is not a power of 2. */
 	if (num_pages != ((u32)1 << (lockaddr_size_log2 - PAGE_SHIFT)))
 		lockaddr_size_log2 += 1;

 	/* Round up if some memory pages spill into the next region. */
 	region_frame_number_start = pfn >> (lockaddr_size_log2 - PAGE_SHIFT);
 	region_frame_number_end =
 	    (pfn + num_pages - 1) >> (lockaddr_size_log2 - PAGE_SHIFT);

 	if (region_frame_number_start < region_frame_number_end)
 		lockaddr_size_log2 += 1;

 	/* Represent the size according to the HW specification. */
 	lockaddr_size_log2 = MAX(lockaddr_size_log2,
 		KBASE_LOCK_REGION_MIN_SIZE_LOG2);

 	if (lockaddr_size_log2 > KBASE_LOCK_REGION_MAX_SIZE_LOG2)
 		return -EINVAL;

 	/* The lowest bits are cleared and then set to size - 1 to represent
 	 * the size in a way that is compatible with the HW specification.
 	 */
 	*lockaddr = lockaddr_base & ~((1ull << lockaddr_size_log2) - 1);
 	*lockaddr |= lockaddr_size_log2 - 1;

 	return 0;
 }

 static int wait_ready(struct kbase_device *kbdev,
 		unsigned int as_nr)
 {
 	unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS;
 	u32 val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS));

 	/* Wait for the MMU status to indicate there is no active command, in
 	 * case one is pending. Do not log remaining register accesses.
 	 */
 	while (--max_loops && (val & AS_STATUS_AS_ACTIVE))
 		val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS));

 	if (max_loops == 0) {
 		dev_err(kbdev->dev, "AS_ACTIVE bit stuck, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n");
 		return -1;
 	}

 	/* If waiting in loop was performed, log last read value. */
 	if (KBASE_AS_INACTIVE_MAX_LOOPS - 1 > max_loops)
 		kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS));

 	return 0;
 }

 static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd)
 {
 	int status;

 	/* write AS_COMMAND when MMU is ready to accept another command */
 	status = wait_ready(kbdev, as_nr);
 	if (status == 0)
 		kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd);

 	return status;
 }

 void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as)
 {
 	struct kbase_mmu_setup *current_setup = &as->current_setup;
 	u64 transcfg = 0;

 	transcfg = current_setup->transcfg;

 	/* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK
 	 * Clear PTW_MEMATTR bits
 	 */
 	transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK;
 	/* Enable correct PTW_MEMATTR bits */
 	transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK;
 	/* Ensure page-tables reads use read-allocate cache-policy in
 	 * the L2
 	 */
 	transcfg |= AS_TRANSCFG_R_ALLOCATE;

 	if (kbdev->system_coherency != COHERENCY_NONE) {
 		/* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable)
 		 * Clear PTW_SH bits
 		 */
 		transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK);
 		/* Enable correct PTW_SH bits */
 		transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS);
 	}

 	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO),
 			transcfg);
 	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI),
 			(transcfg >> 32) & 0xFFFFFFFFUL);

 	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO),
 			current_setup->transtab & 0xFFFFFFFFUL);
 	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI),
 			(current_setup->transtab >> 32) & 0xFFFFFFFFUL);

 	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO),
 			current_setup->memattr & 0xFFFFFFFFUL);
 	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI),
 			(current_setup->memattr >> 32) & 0xFFFFFFFFUL);

 	KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(kbdev, as,
 			current_setup->transtab,
 			current_setup->memattr,
 			transcfg);

 	write_cmd(kbdev, as->number, AS_COMMAND_UPDATE);
 }

 int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as,
 		u64 vpfn, u32 nr, u32 op,
 		unsigned int handling_irq)
 {
 	int ret;

 	lockdep_assert_held(&kbdev->mmu_hw_mutex);

 	if (op == AS_COMMAND_UNLOCK) {
 		/* Unlock doesn't require a lock first */
 		ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK);
 	} else {
 		u64 lock_addr;

 		ret = lock_region(vpfn, nr, &lock_addr);

 		if (!ret) {
 			/* Lock the region that needs to be updated */
 			kbase_reg_write(kbdev,
 				MMU_AS_REG(as->number, AS_LOCKADDR_LO),
 				lock_addr & 0xFFFFFFFFUL);
 			kbase_reg_write(kbdev,
 				MMU_AS_REG(as->number, AS_LOCKADDR_HI),
 				(lock_addr >> 32) & 0xFFFFFFFFUL);
 			write_cmd(kbdev, as->number, AS_COMMAND_LOCK);

 			/* Run the MMU operation */
 			write_cmd(kbdev, as->number, op);

 			/* Wait for the flush to complete */
 			ret = wait_ready(kbdev, as->number);
 		}
 	}

 	return ret;
 }

 void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as,
 		enum kbase_mmu_fault_type type)
 {
 	unsigned long flags;
 	u32 pf_bf_mask;

 	spin_lock_irqsave(&kbdev->mmu_mask_change, flags);

 	/*
 	 * A reset is in-flight and we're flushing the IRQ + bottom half
 	 * so don't update anything as it could race with the reset code.
 	 */
 	if (kbdev->irq_reset_flush)
 		goto unlock;

 	/* Clear the page (and bus fault IRQ as well in case one occurred) */
 	pf_bf_mask = MMU_PAGE_FAULT(as->number);
 #if !MALI_USE_CSF
 	if (type == KBASE_MMU_FAULT_TYPE_BUS ||
 			type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED)
 		pf_bf_mask |= MMU_BUS_ERROR(as->number);
 #endif
 	kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask);

 unlock:
 	spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags);
 }

 void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as,
 		enum kbase_mmu_fault_type type)
 {
 	unsigned long flags;
 	u32 irq_mask;

 	/* Enable the page fault IRQ
 	 * (and bus fault IRQ as well in case one occurred)
 	 */
 	spin_lock_irqsave(&kbdev->mmu_mask_change, flags);

 	/*
 	 * A reset is in-flight and we're flushing the IRQ + bottom half
 	 * so don't update anything as it could race with the reset code.
 	 */
 	if (kbdev->irq_reset_flush)
 		goto unlock;

 	irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)) |
 			MMU_PAGE_FAULT(as->number);

 #if !MALI_USE_CSF
 	if (type == KBASE_MMU_FAULT_TYPE_BUS ||
 			type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED)
 		irq_mask |= MMU_BUS_ERROR(as->number);
 #endif
 	kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask);

 unlock:
 	spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags);
 }
	// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
	/*
	*
	* (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved.
	*
	* This program is free software and is provided to you under the terms of the
	* GNU General Public License version 2 as published by the Free Software
	* Foundation, and any use by you of this program is subject to the terms
	* of such GNU license.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public License
	* along with this program; if not, you can access it online at
	* http://www.gnu.org/licenses/gpl-2.0.html.
	*
	*/

	#include <linux/bitops.h>
	#include <mali_kbase.h>
	#include <mali_kbase_mem.h>
	#include <mmu/mali_kbase_mmu_hw.h>
	#include <tl/mali_kbase_tracepoints.h>
	#include <device/mali_kbase_device.h>

	/**
	* lock_region() - Generate lockaddr to lock memory region in MMU
	* @pfn: Starting page frame number of the region to lock
	* @num_pages: Number of pages to lock. It must be greater than 0.
	* @lockaddr: Address and size of memory region to lock
	*
	* The lockaddr value is a combination of the starting address and
	* the size of the region that encompasses all the memory pages to lock.
	*
	* The size is expressed as a logarithm: it is represented in a way
	* that is compatible with the HW specification and it also determines
	* how many of the lowest bits of the address are cleared.
	*
	* Return: 0 if success, or an error code on failure.
	*/
	static int lock_region(u64 pfn, u32 num_pages, u64 *lockaddr)
	{
	const u64 lockaddr_base = pfn << PAGE_SHIFT;
	u64 lockaddr_size_log2, region_frame_number_start,
	region_frame_number_end;

	if (num_pages == 0)
	return -EINVAL;

	/* The size is expressed as a logarithm and should take into account
	* the possibility that some pages might spill into the next region.
	*/
	lockaddr_size_log2 = fls(num_pages) + PAGE_SHIFT - 1;

	/* Round up if the number of pages is not a power of 2. */
	if (num_pages != ((u32)1 << (lockaddr_size_log2 - PAGE_SHIFT)))
	lockaddr_size_log2 += 1;

	/* Round up if some memory pages spill into the next region. */
	region_frame_number_start = pfn >> (lockaddr_size_log2 - PAGE_SHIFT);
	region_frame_number_end =
	(pfn + num_pages - 1) >> (lockaddr_size_log2 - PAGE_SHIFT);

	if (region_frame_number_start < region_frame_number_end)
	lockaddr_size_log2 += 1;

	/* Represent the size according to the HW specification. */
	lockaddr_size_log2 = MAX(lockaddr_size_log2,
	KBASE_LOCK_REGION_MIN_SIZE_LOG2);

	if (lockaddr_size_log2 > KBASE_LOCK_REGION_MAX_SIZE_LOG2)
	return -EINVAL;

	/* The lowest bits are cleared and then set to size - 1 to represent
	* the size in a way that is compatible with the HW specification.
	*/
	*lockaddr = lockaddr_base & ~((1ull << lockaddr_size_log2) - 1);
	*lockaddr \|= lockaddr_size_log2 - 1;

	return 0;
	}

	static int wait_ready(struct kbase_device *kbdev,
	unsigned int as_nr)
	{
	unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS;
	u32 val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS));

	/* Wait for the MMU status to indicate there is no active command, in
	* case one is pending. Do not log remaining register accesses.
	*/
	while (--max_loops && (val & AS_STATUS_AS_ACTIVE))
	val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS));

	if (max_loops == 0) {
	dev_err(kbdev->dev, "AS_ACTIVE bit stuck, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n");
	return -1;
	}

	/* If waiting in loop was performed, log last read value. */
	if (KBASE_AS_INACTIVE_MAX_LOOPS - 1 > max_loops)
	kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS));

	return 0;
	}

	static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd)
	{
	int status;

	/* write AS_COMMAND when MMU is ready to accept another command */
	status = wait_ready(kbdev, as_nr);
	if (status == 0)
	kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd);

	return status;
	}

	void kbase_mmu_hw_configure(struct kbase_device kbdev, struct kbase_as as)
	{
	struct kbase_mmu_setup *current_setup = &as->current_setup;
	u64 transcfg = 0;

	transcfg = current_setup->transcfg;

	/* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK
	* Clear PTW_MEMATTR bits
	*/
	transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK;
	/* Enable correct PTW_MEMATTR bits */
	transcfg \|= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK;
	/* Ensure page-tables reads use read-allocate cache-policy in
	* the L2
	*/
	transcfg \|= AS_TRANSCFG_R_ALLOCATE;

	if (kbdev->system_coherency != COHERENCY_NONE) {
	/* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable)
	* Clear PTW_SH bits
	*/
	transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK);
	/* Enable correct PTW_SH bits */
	transcfg = (transcfg \| AS_TRANSCFG_PTW_SH_OS);
	}

	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO),
	transcfg);
	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI),
	(transcfg >> 32) & 0xFFFFFFFFUL);

	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO),
	current_setup->transtab & 0xFFFFFFFFUL);
	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI),
	(current_setup->transtab >> 32) & 0xFFFFFFFFUL);

	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO),
	current_setup->memattr & 0xFFFFFFFFUL);
	kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI),
	(current_setup->memattr >> 32) & 0xFFFFFFFFUL);

	KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(kbdev, as,
	current_setup->transtab,
	current_setup->memattr,
	transcfg);

	write_cmd(kbdev, as->number, AS_COMMAND_UPDATE);
	}

	int kbase_mmu_hw_do_operation(struct kbase_device kbdev, struct kbase_as as,
	u64 vpfn, u32 nr, u32 op,
	unsigned int handling_irq)
	{
	int ret;

	lockdep_assert_held(&kbdev->mmu_hw_mutex);

	if (op == AS_COMMAND_UNLOCK) {
	/* Unlock doesn't require a lock first */
	ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK);
	} else {
	u64 lock_addr;

	ret = lock_region(vpfn, nr, &lock_addr);

	if (!ret) {
	/* Lock the region that needs to be updated */
	kbase_reg_write(kbdev,
	MMU_AS_REG(as->number, AS_LOCKADDR_LO),
	lock_addr & 0xFFFFFFFFUL);
	kbase_reg_write(kbdev,
	MMU_AS_REG(as->number, AS_LOCKADDR_HI),
	(lock_addr >> 32) & 0xFFFFFFFFUL);
	write_cmd(kbdev, as->number, AS_COMMAND_LOCK);

	/* Run the MMU operation */
	write_cmd(kbdev, as->number, op);

	/* Wait for the flush to complete */
	ret = wait_ready(kbdev, as->number);
	}
	}

	return ret;
	}

	void kbase_mmu_hw_clear_fault(struct kbase_device kbdev, struct kbase_as as,
	enum kbase_mmu_fault_type type)
	{
	unsigned long flags;
	u32 pf_bf_mask;

	spin_lock_irqsave(&kbdev->mmu_mask_change, flags);

	/*
	* A reset is in-flight and we're flushing the IRQ + bottom half
	* so don't update anything as it could race with the reset code.
	*/
	if (kbdev->irq_reset_flush)
	goto unlock;

	/* Clear the page (and bus fault IRQ as well in case one occurred) */
	pf_bf_mask = MMU_PAGE_FAULT(as->number);
	#if !MALI_USE_CSF
	if (type == KBASE_MMU_FAULT_TYPE_BUS \|\|
	type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED)
	pf_bf_mask \|= MMU_BUS_ERROR(as->number);
	#endif
	kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask);

	unlock:
	spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags);
	}

	void kbase_mmu_hw_enable_fault(struct kbase_device kbdev, struct kbase_as as,
	enum kbase_mmu_fault_type type)
	{
	unsigned long flags;
	u32 irq_mask;

	/* Enable the page fault IRQ
	* (and bus fault IRQ as well in case one occurred)
	*/
	spin_lock_irqsave(&kbdev->mmu_mask_change, flags);

	/*
	* A reset is in-flight and we're flushing the IRQ + bottom half
	* so don't update anything as it could race with the reset code.
	*/
	if (kbdev->irq_reset_flush)
	goto unlock;

	irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)) \|
	MMU_PAGE_FAULT(as->number);

	#if !MALI_USE_CSF
	if (type == KBASE_MMU_FAULT_TYPE_BUS \|\|
	type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED)
	irq_mask \|= MMU_BUS_ERROR(as->number);
	#endif
	kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask);

	unlock:
	spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags);
	}