Project import generated by Copybara. GitOrigin-RevId: dd2e0869181e15d33ff306c4fcc91073a73211a5
diff --git a/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.c b/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.c index b63537f..c2b1892 100644 --- a/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.c +++ b/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.c
@@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2022 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -129,22 +129,13 @@ /* creating a context is considered a disjoint event */ kbase_disjoint_event(kctx->kbdev); + spin_lock_init(&kctx->mm_update_lock); kctx->process_mm = NULL; atomic_set(&kctx->nonmapped_pages, 0); atomic_set(&kctx->permanent_mapped_pages, 0); kctx->tgid = current->tgid; kctx->pid = current->pid; - /* Check if this is a Userspace created context */ - if (likely(kctx->filp)) { - /* This merely takes a reference on the mm_struct and not on the - * address space and so won't block the freeing of address space - * on process exit. - */ - kbase_mem_mmgrab(); - kctx->process_mm = current->mm; - } - atomic_set(&kctx->used_pages, 0); mutex_init(&kctx->reg_lock); @@ -177,11 +168,9 @@ mutex_lock(&kctx->kbdev->kctx_list_lock); err = kbase_insert_kctx_to_process(kctx); - if (err) { - dev_err(kctx->kbdev->dev, "(err:%d) failed to insert kctx to kbase_process", err); - if (likely(kctx->filp)) - mmdrop(kctx->process_mm); - } + if (err) + dev_err(kctx->kbdev->dev, + "(err:%d) failed to insert kctx to kbase_process\n", err); mutex_unlock(&kctx->kbdev->kctx_list_lock); @@ -269,9 +258,6 @@ kbase_remove_kctx_from_process(kctx); mutex_unlock(&kctx->kbdev->kctx_list_lock); - if (likely(kctx->filp)) - mmdrop(kctx->process_mm); - KBASE_KTRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, 0u); }
diff --git a/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h b/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h index 3bd8c4b..c5dd8af 100644 --- a/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h +++ b/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h
@@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2022 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -1590,13 +1590,11 @@ * is scheduled in and an atom is pulled from the context's per * slot runnable tree in JM GPU or GPU command queue * group is programmed on CSG slot in CSF GPU. + * @mm_update_lock: lock used for handling of special tracking page. * @process_mm: Pointer to the memory descriptor of the process which * created the context. Used for accounting the physical * pages used for GPU allocations, done for the context, - * to the memory consumed by the process. A reference is taken - * on this descriptor for the Userspace created contexts so that - * Kbase can safely access it to update the memory usage counters. - * The reference is dropped on context termination. + * to the memory consumed by the process. * @gpu_va_end: End address of the GPU va space (in 4KB page units) * @running_total_tiler_heap_nr_chunks: Running total of number of chunks in all * tiler heaps of the kbase context. @@ -1817,7 +1815,8 @@ atomic_t refcount; - struct mm_struct *process_mm; + spinlock_t mm_update_lock; + struct mm_struct __rcu *process_mm; u64 gpu_va_end; #if MALI_USE_CSF u32 running_total_tiler_heap_nr_chunks;
diff --git a/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h b/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h index 1aebde8..29bf30f 100644 --- a/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h +++ b/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h
@@ -2211,7 +2211,8 @@ * kbase_mem_allow_alloc - Check if allocation of GPU memory is allowed * @kctx: Pointer to kbase context * - * Don't allow the allocation of GPU memory if the ioctl has been issued + * Don't allow the allocation of GPU memory until user space has set up the + * tracking page (which sets kctx->process_mm) or if the ioctl has been issued * from the forked child process using the mali device file fd inherited from * the parent process. * @@ -2219,7 +2220,13 @@ */ static inline bool kbase_mem_allow_alloc(struct kbase_context *kctx) { - return (kctx->process_mm == current->mm); + bool allow_alloc = true; + + rcu_read_lock(); + allow_alloc = (rcu_dereference(kctx->process_mm) == current->mm); + rcu_read_unlock(); + + return allow_alloc; } /**
diff --git a/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c b/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c index a701ba8..c7b6ecc 100644 --- a/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c +++ b/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c
@@ -3207,29 +3207,79 @@ void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages) { - struct mm_struct *mm = kctx->process_mm; + struct mm_struct *mm; - if (unlikely(!mm)) - return; - - atomic_add(pages, &kctx->nonmapped_pages); + rcu_read_lock(); + mm = rcu_dereference(kctx->process_mm); + if (mm) { + atomic_add(pages, &kctx->nonmapped_pages); #ifdef SPLIT_RSS_COUNTING - kbasep_add_mm_counter(mm, MM_FILEPAGES, pages); + kbasep_add_mm_counter(mm, MM_FILEPAGES, pages); +#else + spin_lock(&mm->page_table_lock); + kbasep_add_mm_counter(mm, MM_FILEPAGES, pages); + spin_unlock(&mm->page_table_lock); +#endif + } + rcu_read_unlock(); +} + +static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx) +{ + int pages; + struct mm_struct *mm; + + spin_lock(&kctx->mm_update_lock); + mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock)); + if (!mm) { + spin_unlock(&kctx->mm_update_lock); + return; + } + + rcu_assign_pointer(kctx->process_mm, NULL); + spin_unlock(&kctx->mm_update_lock); + synchronize_rcu(); + + pages = atomic_xchg(&kctx->nonmapped_pages, 0); +#ifdef SPLIT_RSS_COUNTING + kbasep_add_mm_counter(mm, MM_FILEPAGES, -pages); #else spin_lock(&mm->page_table_lock); - kbasep_add_mm_counter(mm, MM_FILEPAGES, pages); + kbasep_add_mm_counter(mm, MM_FILEPAGES, -pages); spin_unlock(&mm->page_table_lock); #endif } +static void kbase_special_vm_close(struct vm_area_struct *vma) +{ + struct kbase_context *kctx; + + kctx = vma->vm_private_data; + kbasep_os_process_page_usage_drain(kctx); +} + +static const struct vm_operations_struct kbase_vm_special_ops = { + .close = kbase_special_vm_close, +}; + static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma) { - if (vma_pages(vma) != 1) - return -EINVAL; + /* check that this is the only tracking page */ + spin_lock(&kctx->mm_update_lock); + if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) { + spin_unlock(&kctx->mm_update_lock); + return -EFAULT; + } + + rcu_assign_pointer(kctx->process_mm, current->mm); + + spin_unlock(&kctx->mm_update_lock); /* no real access */ vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; + vma->vm_ops = &kbase_vm_special_ops; + vma->vm_private_data = kctx; return 0; }
diff --git a/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h b/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h index 75ec115..5b12e18 100644 --- a/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h +++ b/bifrost/r38p2/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h
@@ -501,21 +501,6 @@ */ u32 kbase_get_cache_line_alignment(struct kbase_device *kbdev); -/** - * kbase_mem_mmgrab - Wrapper function to take reference on mm_struct of current process - */ -static inline void kbase_mem_mmgrab(void) -{ - /* This merely takes a reference on the memory descriptor structure - * i.e. mm_struct of current process and not on its address space and - * so won't block the freeing of address space on process exit. - */ -#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - atomic_inc(¤t->mm->mm_count); -#else - mmgrab(current->mm); -#endif -} #if (KERNEL_VERSION(4, 20, 0) > LINUX_VERSION_CODE) static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot)
diff --git a/bifrost/r38p2/kernel/include/uapi/gpu/arm/midgard/csf/mali_kbase_csf_ioctl.h b/bifrost/r38p2/kernel/include/uapi/gpu/arm/midgard/csf/mali_kbase_csf_ioctl.h index ec21c53..cbb7310 100644 --- a/bifrost/r38p2/kernel/include/uapi/gpu/arm/midgard/csf/mali_kbase_csf_ioctl.h +++ b/bifrost/r38p2/kernel/include/uapi/gpu/arm/midgard/csf/mali_kbase_csf_ioctl.h
@@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2022 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -64,13 +64,10 @@ * - Added ioctl to query a register of USER page. * 1.14: * - Added support for passing down the buffer descriptor VA in tiler heap init - * 1.18 - * - Relax the requirement to create a mapping with BASE_MEM_MAP_TRACKING_HANDLE - * before allocating GPU memory for the context. */ #define BASE_UK_VERSION_MAJOR 1 -#define BASE_UK_VERSION_MINOR 18 +#define BASE_UK_VERSION_MINOR 14 /** * struct kbase_ioctl_version_check - Check version compatibility between
diff --git a/bifrost/r38p2/kernel/include/uapi/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h b/bifrost/r38p2/kernel/include/uapi/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h index 2cc2f63..20d931a 100644 --- a/bifrost/r38p2/kernel/include/uapi/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h +++ b/bifrost/r38p2/kernel/include/uapi/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h
@@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2022 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -127,12 +127,9 @@ * - First release of new HW performance counters interface. * 11.35: * - Dummy model (no mali) backend will now clear HWC values after each sample - * 11.38: - * - Relax the requirement to create a mapping with BASE_MEM_MAP_TRACKING_HANDLE - * before allocating GPU memory for the context. */ #define BASE_UK_VERSION_MAJOR 11 -#define BASE_UK_VERSION_MINOR 38 +#define BASE_UK_VERSION_MINOR 35 /** * struct kbase_ioctl_version_check - Check version compatibility between
diff --git a/bifrost/r43p0/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c b/bifrost/r43p0/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c index 0668ab4..bff26b1 100644 --- a/bifrost/r43p0/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c +++ b/bifrost/r43p0/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c
@@ -1410,7 +1410,6 @@ kbase_gpu_vm_unlock(kctx); } else { int ret = -ENOMEM; - const u8 group_id = region->gpu_alloc->group_id; kbase_gpu_vm_unlock(kctx); @@ -1421,7 +1420,8 @@ if (kbdev->pagesize_2mb && grow_2mb_pool) { /* Round page requirement up to nearest 2 MB */ struct kbase_mem_pool *const lp_mem_pool = - &kctx->mem_pools.large[group_id]; + &kctx->mem_pools.large[ + region->gpu_alloc->group_id]; pages_to_grow = (pages_to_grow + ((1 << lp_mem_pool->order) - 1)) @@ -1431,7 +1431,8 @@ pages_to_grow, kctx->task); } else { struct kbase_mem_pool *const mem_pool = - &kctx->mem_pools.small[group_id]; + &kctx->mem_pools.small[ + region->gpu_alloc->group_id]; ret = kbase_mem_pool_grow(mem_pool, pages_to_grow, kctx->task);