bifrost/r12p0/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c - manifest_repos/mali-driver - Git at Google

 /*
  *
  * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
  * Foundation, and any use by you of this program is subject to the terms
  * of such GNU licence.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, you can access it online at
  * http://www.gnu.org/licenses/gpl-2.0.html.
  *
  * SPDX-License-Identifier: GPL-2.0
  *
  */

 #include <mali_kbase.h>
 #include <linux/mm.h>
 #include <linux/dma-mapping.h>
 #include <linux/highmem.h>
 #include <linux/spinlock.h>
 #include <linux/shrinker.h>
 #include <linux/atomic.h>
 #include <linux/version.h>

 #define pool_dbg(pool, format, ...) \
 	dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format,	\
 		(pool->next_pool) ? "kctx" : "kbdev",	\
 		kbase_mem_pool_size(pool),	\
 		kbase_mem_pool_max_size(pool),	\
 		##__VA_ARGS__)

 #define NOT_DIRTY false
 #define NOT_RECLAIMED false

 static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool)
 {
 	ssize_t max_size = kbase_mem_pool_max_size(pool);
 	ssize_t cur_size = kbase_mem_pool_size(pool);

 	return max(max_size - cur_size, (ssize_t)0);
 }

 static bool kbase_mem_pool_is_full(struct kbase_mem_pool *pool)
 {
 	return kbase_mem_pool_size(pool) >= kbase_mem_pool_max_size(pool);
 }

 static bool kbase_mem_pool_is_empty(struct kbase_mem_pool *pool)
 {
 	return kbase_mem_pool_size(pool) == 0;
 }

 static void kbase_mem_pool_add_locked(struct kbase_mem_pool *pool,
 		struct page *p)
 {
 	lockdep_assert_held(&pool->pool_lock);

 	list_add(&p->lru, &pool->page_list);
 	pool->cur_size++;

 	pool_dbg(pool, "added page\n");
 }

 static void kbase_mem_pool_add(struct kbase_mem_pool *pool, struct page *p)
 {
 	kbase_mem_pool_lock(pool);
 	kbase_mem_pool_add_locked(pool, p);
 	kbase_mem_pool_unlock(pool);
 }

 static void kbase_mem_pool_add_list_locked(struct kbase_mem_pool *pool,
 		struct list_head *page_list, size_t nr_pages)
 {
 	lockdep_assert_held(&pool->pool_lock);

 	list_splice(page_list, &pool->page_list);
 	pool->cur_size += nr_pages;

 	pool_dbg(pool, "added %zu pages\n", nr_pages);
 }

 static void kbase_mem_pool_add_list(struct kbase_mem_pool *pool,
 		struct list_head *page_list, size_t nr_pages)
 {
 	kbase_mem_pool_lock(pool);
 	kbase_mem_pool_add_list_locked(pool, page_list, nr_pages);
 	kbase_mem_pool_unlock(pool);
 }

 static struct page *kbase_mem_pool_remove_locked(struct kbase_mem_pool *pool)
 {
 	struct page *p;

 	lockdep_assert_held(&pool->pool_lock);

 	if (kbase_mem_pool_is_empty(pool))
 		return NULL;

 	p = list_first_entry(&pool->page_list, struct page, lru);
 	list_del_init(&p->lru);
 	pool->cur_size--;

 	pool_dbg(pool, "removed page\n");

 	return p;
 }

 static struct page *kbase_mem_pool_remove(struct kbase_mem_pool *pool)
 {
 	struct page *p;

 	kbase_mem_pool_lock(pool);
 	p = kbase_mem_pool_remove_locked(pool);
 	kbase_mem_pool_unlock(pool);

 	return p;
 }

 static void kbase_mem_pool_sync_page(struct kbase_mem_pool *pool,
 		struct page *p)
 {
 	struct device *dev = pool->kbdev->dev;
 	dma_sync_single_for_device(dev, kbase_dma_addr(p),
 			(PAGE_SIZE << pool->order), DMA_BIDIRECTIONAL);
 }

 static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool,
 		struct page *p)
 {
 	int i;

 	for (i = 0; i < (1U << pool->order); i++)
 		clear_highpage(p+i);

 	kbase_mem_pool_sync_page(pool, p);
 }

 static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool,
 		struct page *p)
 {
 	/* Zero page before spilling */
 	kbase_mem_pool_zero_page(next_pool, p);

 	kbase_mem_pool_add(next_pool, p);
 }

 struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool)
 {
 	struct page *p;
 	gfp_t gfp;
 	struct device *dev = pool->kbdev->dev;
 	dma_addr_t dma_addr;
 	int i;

 #if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \
 	LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0)
 	/* DMA cache sync fails for HIGHMEM before 3.5 on ARM */
 	gfp = GFP_USER | __GFP_ZERO;
 #else
 	gfp = GFP_HIGHUSER | __GFP_ZERO;
 #endif

 	/* don't warn on higer order failures */
 	if (pool->order)
 		gfp |= __GFP_NOWARN;

 	p = alloc_pages(gfp, pool->order);
 	if (!p)
 		return NULL;

 	dma_addr = dma_map_page(dev, p, 0, (PAGE_SIZE << pool->order),
 				DMA_BIDIRECTIONAL);
 	if (dma_mapping_error(dev, dma_addr)) {
 		__free_pages(p, pool->order);
 		return NULL;
 	}

 	WARN_ON(dma_addr != page_to_phys(p));
 	for (i = 0; i < (1u << pool->order); i++)
 		kbase_set_dma_addr(p+i, dma_addr + PAGE_SIZE * i);

 	return p;
 }

 static void kbase_mem_pool_free_page(struct kbase_mem_pool *pool,
 		struct page *p)
 {
 	struct device *dev = pool->kbdev->dev;
 	dma_addr_t dma_addr = kbase_dma_addr(p);
 	int i;

 	dma_unmap_page(dev, dma_addr, (PAGE_SIZE << pool->order),
 		       DMA_BIDIRECTIONAL);
 	for (i = 0; i < (1u << pool->order); i++)
 		kbase_clear_dma_addr(p+i);
 	__free_pages(p, pool->order);

 	pool_dbg(pool, "freed page to kernel\n");
 }

 static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool,
 		size_t nr_to_shrink)
 {
 	struct page *p;
 	size_t i;

 	lockdep_assert_held(&pool->pool_lock);

 	for (i = 0; i < nr_to_shrink && !kbase_mem_pool_is_empty(pool); i++) {
 		p = kbase_mem_pool_remove_locked(pool);
 		kbase_mem_pool_free_page(pool, p);
 	}

 	return i;
 }

 static size_t kbase_mem_pool_shrink(struct kbase_mem_pool *pool,
 		size_t nr_to_shrink)
 {
 	size_t nr_freed;

 	kbase_mem_pool_lock(pool);
 	nr_freed = kbase_mem_pool_shrink_locked(pool, nr_to_shrink);
 	kbase_mem_pool_unlock(pool);

 	return nr_freed;
 }

 int kbase_mem_pool_grow(struct kbase_mem_pool *pool,
 		size_t nr_to_grow)
 {
 	struct page *p;
 	size_t i;

 	kbase_mem_pool_lock(pool);

 	pool->dont_reclaim = true;
 	for (i = 0; i < nr_to_grow; i++) {
 		if (pool->dying) {
 			pool->dont_reclaim = false;
 			kbase_mem_pool_shrink_locked(pool, nr_to_grow);
 			kbase_mem_pool_unlock(pool);

 			return -ENOMEM;
 		}
 		kbase_mem_pool_unlock(pool);

 		p = kbase_mem_alloc_page(pool);
 		if (!p) {
 			kbase_mem_pool_lock(pool);
 			pool->dont_reclaim = false;
 			kbase_mem_pool_unlock(pool);

 			return -ENOMEM;
 		}

 		kbase_mem_pool_lock(pool);
 		kbase_mem_pool_add_locked(pool, p);
 	}
 	pool->dont_reclaim = false;
 	kbase_mem_pool_unlock(pool);

 	return 0;
 }

 void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size)
 {
 	size_t cur_size;
 	int err = 0;

 	cur_size = kbase_mem_pool_size(pool);

 	if (new_size > pool->max_size)
 		new_size = pool->max_size;

 	if (new_size < cur_size)
 		kbase_mem_pool_shrink(pool, cur_size - new_size);
 	else if (new_size > cur_size)
 		err = kbase_mem_pool_grow(pool, new_size - cur_size);

 	if (err) {
 		size_t grown_size = kbase_mem_pool_size(pool);

 		dev_warn(pool->kbdev->dev,
 			 "Mem pool not grown to the required size of %zu bytes, grown for additional %zu bytes instead!\n",
 			 (new_size - cur_size), (grown_size - cur_size));
 	}
 }

 void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size)
 {
 	size_t cur_size;
 	size_t nr_to_shrink;

 	kbase_mem_pool_lock(pool);

 	pool->max_size = max_size;

 	cur_size = kbase_mem_pool_size(pool);
 	if (max_size < cur_size) {
 		nr_to_shrink = cur_size - max_size;
 		kbase_mem_pool_shrink_locked(pool, nr_to_shrink);
 	}

 	kbase_mem_pool_unlock(pool);
 }


 static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s,
 		struct shrink_control *sc)
 {
 	struct kbase_mem_pool *pool;
 	size_t pool_size;

 	pool = container_of(s, struct kbase_mem_pool, reclaim);

 	kbase_mem_pool_lock(pool);
 	if (pool->dont_reclaim && !pool->dying) {
 		kbase_mem_pool_unlock(pool);
 		return 0;
 	}
 	pool_size = kbase_mem_pool_size(pool);
 	kbase_mem_pool_unlock(pool);

 	return pool_size;
 }

 static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s,
 		struct shrink_control *sc)
 {
 	struct kbase_mem_pool *pool;
 	unsigned long freed;

 	pool = container_of(s, struct kbase_mem_pool, reclaim);

 	kbase_mem_pool_lock(pool);
 	if (pool->dont_reclaim && !pool->dying) {
 		kbase_mem_pool_unlock(pool);
 		return 0;
 	}

 	pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan);

 	freed = kbase_mem_pool_shrink_locked(pool, sc->nr_to_scan);

 	kbase_mem_pool_unlock(pool);

 	pool_dbg(pool, "reclaim freed %ld pages\n", freed);

 	return freed;
 }

 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
 static int kbase_mem_pool_reclaim_shrink(struct shrinker *s,
 		struct shrink_control *sc)
 {
 	if (sc->nr_to_scan == 0)
 		return kbase_mem_pool_reclaim_count_objects(s, sc);

 	return kbase_mem_pool_reclaim_scan_objects(s, sc);
 }
 #endif

 int kbase_mem_pool_init(struct kbase_mem_pool *pool,
 		size_t max_size,
 		size_t order,
 		struct kbase_device *kbdev,
 		struct kbase_mem_pool *next_pool)
 {
 	pool->cur_size = 0;
 	pool->max_size = max_size;
 	pool->order = order;
 	pool->kbdev = kbdev;
 	pool->next_pool = next_pool;
 	pool->dying = false;

 	spin_lock_init(&pool->pool_lock);
 	INIT_LIST_HEAD(&pool->page_list);

 	/* Register shrinker */
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
 	pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink;
 #else
 	pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects;
 	pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects;
 #endif
 	pool->reclaim.seeks = DEFAULT_SEEKS;
 	/* Kernel versions prior to 3.1 :
 	 * struct shrinker does not define batch */
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0)
 	pool->reclaim.batch = 0;
 #endif
 	register_shrinker(&pool->reclaim);

 	pool_dbg(pool, "initialized\n");

 	return 0;
 }

 void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool)
 {
 	kbase_mem_pool_lock(pool);
 	pool->dying = true;
 	kbase_mem_pool_unlock(pool);
 }

 void kbase_mem_pool_term(struct kbase_mem_pool *pool)
 {
 	struct kbase_mem_pool *next_pool = pool->next_pool;
 	struct page *p;
 	size_t nr_to_spill = 0;
 	LIST_HEAD(spill_list);
 	int i;

 	pool_dbg(pool, "terminate()\n");

 	unregister_shrinker(&pool->reclaim);

 	kbase_mem_pool_lock(pool);
 	pool->max_size = 0;

 	if (next_pool && !kbase_mem_pool_is_full(next_pool)) {
 		/* Spill to next pool (may overspill) */
 		nr_to_spill = kbase_mem_pool_capacity(next_pool);
 		nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill);

 		/* Zero pages first without holding the next_pool lock */
 		for (i = 0; i < nr_to_spill; i++) {
 			p = kbase_mem_pool_remove_locked(pool);
 			kbase_mem_pool_zero_page(pool, p);
 			list_add(&p->lru, &spill_list);
 		}
 	}

 	while (!kbase_mem_pool_is_empty(pool)) {
 		/* Free remaining pages to kernel */
 		p = kbase_mem_pool_remove_locked(pool);
 		kbase_mem_pool_free_page(pool, p);
 	}

 	kbase_mem_pool_unlock(pool);

 	if (next_pool && nr_to_spill) {
 		/* Add new page list to next_pool */
 		kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill);

 		pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill);
 	}

 	pool_dbg(pool, "terminated\n");
 }

 struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool)
 {
 	struct page *p;

 	do {
 		pool_dbg(pool, "alloc()\n");
 		p = kbase_mem_pool_remove(pool);

 		if (p)
 			return p;

 		pool = pool->next_pool;
 	} while (pool);

 	return NULL;
 }

 struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool)
 {
 	struct page *p;

 	lockdep_assert_held(&pool->pool_lock);

 	pool_dbg(pool, "alloc_locked()\n");
 	p = kbase_mem_pool_remove_locked(pool);

 	if (p)
 		return p;

 	return NULL;
 }

 void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p,
 		bool dirty)
 {
 	struct kbase_mem_pool *next_pool = pool->next_pool;

 	pool_dbg(pool, "free()\n");

 	if (!kbase_mem_pool_is_full(pool)) {
 		/* Add to our own pool */
 		if (dirty)
 			kbase_mem_pool_sync_page(pool, p);

 		kbase_mem_pool_add(pool, p);
 	} else if (next_pool && !kbase_mem_pool_is_full(next_pool)) {
 		/* Spill to next pool */
 		kbase_mem_pool_spill(next_pool, p);
 	} else {
 		/* Free page */
 		kbase_mem_pool_free_page(pool, p);
 	}
 }

 void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p,
 		bool dirty)
 {
 	pool_dbg(pool, "free_locked()\n");

 	lockdep_assert_held(&pool->pool_lock);

 	if (!kbase_mem_pool_is_full(pool)) {
 		/* Add to our own pool */
 		if (dirty)
 			kbase_mem_pool_sync_page(pool, p);

 		kbase_mem_pool_add_locked(pool, p);
 	} else {
 		/* Free page */
 		kbase_mem_pool_free_page(pool, p);
 	}
 }

 int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages,
 		struct tagged_addr *pages, bool partial_allowed)
 {
 	struct page *p;
 	size_t nr_from_pool;
 	size_t i = 0;
 	int err = -ENOMEM;
 	size_t nr_pages_internal;

 	nr_pages_internal = nr_4k_pages / (1u << (pool->order));

 	if (nr_pages_internal * (1u << pool->order) != nr_4k_pages)
 		return -EINVAL;

 	pool_dbg(pool, "alloc_pages(4k=%zu):\n", nr_4k_pages);
 	pool_dbg(pool, "alloc_pages(internal=%zu):\n", nr_pages_internal);

 	/* Get pages from this pool */
 	kbase_mem_pool_lock(pool);
 	nr_from_pool = min(nr_pages_internal, kbase_mem_pool_size(pool));
 	while (nr_from_pool--) {
 		int j;
 		p = kbase_mem_pool_remove_locked(pool);
 		if (pool->order) {
 			pages[i++] = as_tagged_tag(page_to_phys(p),
 						   HUGE_HEAD | HUGE_PAGE);
 			for (j = 1; j < (1u << pool->order); j++)
 				pages[i++] = as_tagged_tag(page_to_phys(p) +
 							   PAGE_SIZE * j,
 							   HUGE_PAGE);
 		} else {
 			pages[i++] = as_tagged(page_to_phys(p));
 		}
 	}
 	kbase_mem_pool_unlock(pool);

 	if (i != nr_4k_pages && pool->next_pool) {
 		/* Allocate via next pool */
 		err = kbase_mem_pool_alloc_pages(pool->next_pool,
 				nr_4k_pages - i, pages + i, partial_allowed);

 		if (err < 0)
 			goto err_rollback;

 		i += err;
 	} else {
 		/* Get any remaining pages from kernel */
 		while (i != nr_4k_pages) {
 			p = kbase_mem_alloc_page(pool);
 			if (!p) {
 				if (partial_allowed)
 					goto done;
 				else
 					goto err_rollback;
 			}

 			if (pool->order) {
 				int j;

 				pages[i++] = as_tagged_tag(page_to_phys(p),
 							   HUGE_PAGE |
 							   HUGE_HEAD);
 				for (j = 1; j < (1u << pool->order); j++) {
 					phys_addr_t phys;

 					phys = page_to_phys(p) + PAGE_SIZE * j;
 					pages[i++] = as_tagged_tag(phys,
 								   HUGE_PAGE);
 				}
 			} else {
 				pages[i++] = as_tagged(page_to_phys(p));
 			}
 		}
 	}

 done:
 	pool_dbg(pool, "alloc_pages(%zu) done\n", i);
 	return i;

 err_rollback:
 	kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED);
 	return err;
 }

 int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool,
 		size_t nr_4k_pages, struct tagged_addr *pages)
 {
 	struct page *p;
 	size_t i;
 	size_t nr_pages_internal;

 	lockdep_assert_held(&pool->pool_lock);

 	nr_pages_internal = nr_4k_pages / (1u << (pool->order));

 	if (nr_pages_internal * (1u << pool->order) != nr_4k_pages)
 		return -EINVAL;

 	pool_dbg(pool, "alloc_pages_locked(4k=%zu):\n", nr_4k_pages);
 	pool_dbg(pool, "alloc_pages_locked(internal=%zu):\n",
 			nr_pages_internal);

 	if (kbase_mem_pool_size(pool) < nr_pages_internal) {
 		pool_dbg(pool, "Failed alloc\n");
 		return -ENOMEM;
 	}

 	for (i = 0; i < nr_pages_internal; i++) {
 		int j;

 		p = kbase_mem_pool_remove_locked(pool);
 		if (pool->order) {
 			*pages++ = as_tagged_tag(page_to_phys(p),
 						   HUGE_HEAD | HUGE_PAGE);
 			for (j = 1; j < (1u << pool->order); j++) {
 				*pages++ = as_tagged_tag(page_to_phys(p) +
 							   PAGE_SIZE * j,
 							   HUGE_PAGE);
 			}
 		} else {
 			*pages++ = as_tagged(page_to_phys(p));
 		}
 	}

 	return nr_4k_pages;
 }

 static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool,
 				     size_t nr_pages, struct tagged_addr *pages,
 				     bool zero, bool sync)
 {
 	struct page *p;
 	size_t nr_to_pool = 0;
 	LIST_HEAD(new_page_list);
 	size_t i;

 	if (!nr_pages)
 		return;

 	pool_dbg(pool, "add_array(%zu, zero=%d, sync=%d):\n",
 			nr_pages, zero, sync);

 	/* Zero/sync pages first without holding the pool lock */
 	for (i = 0; i < nr_pages; i++) {
 		if (unlikely(!as_phys_addr_t(pages[i])))
 			continue;

 		if (is_huge_head(pages[i]) || !is_huge(pages[i])) {
 			p = phys_to_page(as_phys_addr_t(pages[i]));
 			if (zero)
 				kbase_mem_pool_zero_page(pool, p);
 			else if (sync)
 				kbase_mem_pool_sync_page(pool, p);

 			list_add(&p->lru, &new_page_list);
 			nr_to_pool++;
 		}
 		pages[i] = as_tagged(0);
 	}

 	/* Add new page list to pool */
 	kbase_mem_pool_add_list(pool, &new_page_list, nr_to_pool);

 	pool_dbg(pool, "add_array(%zu) added %zu pages\n",
 			nr_pages, nr_to_pool);
 }

 static void kbase_mem_pool_add_array_locked(struct kbase_mem_pool *pool,
 		size_t nr_pages, struct tagged_addr *pages,
 		bool zero, bool sync)
 {
 	struct page *p;
 	size_t nr_to_pool = 0;
 	LIST_HEAD(new_page_list);
 	size_t i;

 	lockdep_assert_held(&pool->pool_lock);

 	if (!nr_pages)
 		return;

 	pool_dbg(pool, "add_array_locked(%zu, zero=%d, sync=%d):\n",
 			nr_pages, zero, sync);

 	/* Zero/sync pages first */
 	for (i = 0; i < nr_pages; i++) {
 		if (unlikely(!as_phys_addr_t(pages[i])))
 			continue;

 		if (is_huge_head(pages[i]) || !is_huge(pages[i])) {
 			p = phys_to_page(as_phys_addr_t(pages[i]));
 			if (zero)
 				kbase_mem_pool_zero_page(pool, p);
 			else if (sync)
 				kbase_mem_pool_sync_page(pool, p);

 			list_add(&p->lru, &new_page_list);
 			nr_to_pool++;
 		}
 		pages[i] = as_tagged(0);
 	}

 	/* Add new page list to pool */
 	kbase_mem_pool_add_list_locked(pool, &new_page_list, nr_to_pool);

 	pool_dbg(pool, "add_array_locked(%zu) added %zu pages\n",
 			nr_pages, nr_to_pool);
 }

 void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages,
 		struct tagged_addr *pages, bool dirty, bool reclaimed)
 {
 	struct kbase_mem_pool *next_pool = pool->next_pool;
 	struct page *p;
 	size_t nr_to_pool;
 	LIST_HEAD(to_pool_list);
 	size_t i = 0;

 	pool_dbg(pool, "free_pages(%zu):\n", nr_pages);

 	if (!reclaimed) {
 		/* Add to this pool */
 		nr_to_pool = kbase_mem_pool_capacity(pool);
 		nr_to_pool = min(nr_pages, nr_to_pool);

 		kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty);

 		i += nr_to_pool;

 		if (i != nr_pages && next_pool) {
 			/* Spill to next pool (may overspill) */
 			nr_to_pool = kbase_mem_pool_capacity(next_pool);
 			nr_to_pool = min(nr_pages - i, nr_to_pool);

 			kbase_mem_pool_add_array(next_pool, nr_to_pool,
 					pages + i, true, dirty);
 			i += nr_to_pool;
 		}
 	}

 	/* Free any remaining pages to kernel */
 	for (; i < nr_pages; i++) {
 		if (unlikely(!as_phys_addr_t(pages[i])))
 			continue;

 		if (is_huge(pages[i]) && !is_huge_head(pages[i])) {
 			pages[i] = as_tagged(0);
 			continue;
 		}

 		p = phys_to_page(as_phys_addr_t(pages[i]));

 		kbase_mem_pool_free_page(pool, p);
 		pages[i] = as_tagged(0);
 	}

 	pool_dbg(pool, "free_pages(%zu) done\n", nr_pages);
 }


 void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool,
 		size_t nr_pages, struct tagged_addr *pages, bool dirty,
 		bool reclaimed)
 {
 	struct page *p;
 	size_t nr_to_pool;
 	LIST_HEAD(to_pool_list);
 	size_t i = 0;

 	lockdep_assert_held(&pool->pool_lock);

 	pool_dbg(pool, "free_pages_locked(%zu):\n", nr_pages);

 	if (!reclaimed) {
 		/* Add to this pool */
 		nr_to_pool = kbase_mem_pool_capacity(pool);
 		nr_to_pool = min(nr_pages, nr_to_pool);

 		kbase_mem_pool_add_array_locked(pool, nr_pages, pages, false,
 				dirty);

 		i += nr_to_pool;
 	}

 	/* Free any remaining pages to kernel */
 	for (; i < nr_pages; i++) {
 		if (unlikely(!as_phys_addr_t(pages[i])))
 			continue;

 		if (is_huge(pages[i]) && !is_huge_head(pages[i])) {
 			pages[i] = as_tagged(0);
 			continue;
 		}

 		p = phys_to_page(as_phys_addr_t(pages[i]));

 		kbase_mem_pool_free_page(pool, p);
 		pages[i] = as_tagged(0);
 	}

 	pool_dbg(pool, "free_pages_locked(%zu) done\n", nr_pages);
 }
	/*
	*
	* (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved.
	*
	* This program is free software and is provided to you under the terms of the
	* GNU General Public License version 2 as published by the Free Software
	* Foundation, and any use by you of this program is subject to the terms
	* of such GNU licence.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public License
	* along with this program; if not, you can access it online at
	* http://www.gnu.org/licenses/gpl-2.0.html.
	*
	* SPDX-License-Identifier: GPL-2.0
	*
	*/

	#include <mali_kbase.h>
	#include <linux/mm.h>
	#include <linux/dma-mapping.h>
	#include <linux/highmem.h>
	#include <linux/spinlock.h>
	#include <linux/shrinker.h>
	#include <linux/atomic.h>
	#include <linux/version.h>

	#define pool_dbg(pool, format, ...) \
	dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \
	(pool->next_pool) ? "kctx" : "kbdev", \
	kbase_mem_pool_size(pool), \
	kbase_mem_pool_max_size(pool), \
	##__VA_ARGS__)

	#define NOT_DIRTY false
	#define NOT_RECLAIMED false

	static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool)
	{
	ssize_t max_size = kbase_mem_pool_max_size(pool);
	ssize_t cur_size = kbase_mem_pool_size(pool);

	return max(max_size - cur_size, (ssize_t)0);
	}

	static bool kbase_mem_pool_is_full(struct kbase_mem_pool *pool)
	{
	return kbase_mem_pool_size(pool) >= kbase_mem_pool_max_size(pool);
	}

	static bool kbase_mem_pool_is_empty(struct kbase_mem_pool *pool)
	{
	return kbase_mem_pool_size(pool) == 0;
	}

	static void kbase_mem_pool_add_locked(struct kbase_mem_pool *pool,
	struct page *p)
	{
	lockdep_assert_held(&pool->pool_lock);

	list_add(&p->lru, &pool->page_list);
	pool->cur_size++;

	pool_dbg(pool, "added page\n");
	}

	static void kbase_mem_pool_add(struct kbase_mem_pool pool, struct page p)
	{
	kbase_mem_pool_lock(pool);
	kbase_mem_pool_add_locked(pool, p);
	kbase_mem_pool_unlock(pool);
	}

	static void kbase_mem_pool_add_list_locked(struct kbase_mem_pool *pool,
	struct list_head *page_list, size_t nr_pages)
	{
	lockdep_assert_held(&pool->pool_lock);

	list_splice(page_list, &pool->page_list);
	pool->cur_size += nr_pages;

	pool_dbg(pool, "added %zu pages\n", nr_pages);
	}

	static void kbase_mem_pool_add_list(struct kbase_mem_pool *pool,
	struct list_head *page_list, size_t nr_pages)
	{
	kbase_mem_pool_lock(pool);
	kbase_mem_pool_add_list_locked(pool, page_list, nr_pages);
	kbase_mem_pool_unlock(pool);
	}

	static struct page kbase_mem_pool_remove_locked(struct kbase_mem_pool pool)
	{
	struct page *p;

	lockdep_assert_held(&pool->pool_lock);

	if (kbase_mem_pool_is_empty(pool))
	return NULL;

	p = list_first_entry(&pool->page_list, struct page, lru);
	list_del_init(&p->lru);
	pool->cur_size--;

	pool_dbg(pool, "removed page\n");

	return p;
	}

	static struct page kbase_mem_pool_remove(struct kbase_mem_pool pool)
	{
	struct page *p;

	kbase_mem_pool_lock(pool);
	p = kbase_mem_pool_remove_locked(pool);
	kbase_mem_pool_unlock(pool);

	return p;
	}

	static void kbase_mem_pool_sync_page(struct kbase_mem_pool *pool,
	struct page *p)
	{
	struct device *dev = pool->kbdev->dev;
	dma_sync_single_for_device(dev, kbase_dma_addr(p),
	(PAGE_SIZE << pool->order), DMA_BIDIRECTIONAL);
	}

	static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool,
	struct page *p)
	{
	int i;

	for (i = 0; i < (1U << pool->order); i++)
	clear_highpage(p+i);

	kbase_mem_pool_sync_page(pool, p);
	}

	static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool,
	struct page *p)
	{
	/* Zero page before spilling */
	kbase_mem_pool_zero_page(next_pool, p);

	kbase_mem_pool_add(next_pool, p);
	}

	struct page kbase_mem_alloc_page(struct kbase_mem_pool pool)
	{
	struct page *p;
	gfp_t gfp;
	struct device *dev = pool->kbdev->dev;
	dma_addr_t dma_addr;
	int i;

	#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \
	LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0)
	/* DMA cache sync fails for HIGHMEM before 3.5 on ARM */
	gfp = GFP_USER \| __GFP_ZERO;
	#else
	gfp = GFP_HIGHUSER \| __GFP_ZERO;
	#endif

	/* don't warn on higer order failures */
	if (pool->order)
	gfp \|= __GFP_NOWARN;

	p = alloc_pages(gfp, pool->order);
	if (!p)
	return NULL;

	dma_addr = dma_map_page(dev, p, 0, (PAGE_SIZE << pool->order),
	DMA_BIDIRECTIONAL);
	if (dma_mapping_error(dev, dma_addr)) {
	__free_pages(p, pool->order);
	return NULL;
	}

	WARN_ON(dma_addr != page_to_phys(p));
	for (i = 0; i < (1u << pool->order); i++)
	kbase_set_dma_addr(p+i, dma_addr + PAGE_SIZE * i);

	return p;
	}

	static void kbase_mem_pool_free_page(struct kbase_mem_pool *pool,
	struct page *p)
	{
	struct device *dev = pool->kbdev->dev;
	dma_addr_t dma_addr = kbase_dma_addr(p);
	int i;

	dma_unmap_page(dev, dma_addr, (PAGE_SIZE << pool->order),
	DMA_BIDIRECTIONAL);
	for (i = 0; i < (1u << pool->order); i++)
	kbase_clear_dma_addr(p+i);
	__free_pages(p, pool->order);

	pool_dbg(pool, "freed page to kernel\n");
	}

	static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool,
	size_t nr_to_shrink)
	{
	struct page *p;
	size_t i;

	lockdep_assert_held(&pool->pool_lock);

	for (i = 0; i < nr_to_shrink && !kbase_mem_pool_is_empty(pool); i++) {
	p = kbase_mem_pool_remove_locked(pool);
	kbase_mem_pool_free_page(pool, p);
	}

	return i;
	}

	static size_t kbase_mem_pool_shrink(struct kbase_mem_pool *pool,
	size_t nr_to_shrink)
	{
	size_t nr_freed;

	kbase_mem_pool_lock(pool);
	nr_freed = kbase_mem_pool_shrink_locked(pool, nr_to_shrink);
	kbase_mem_pool_unlock(pool);

	return nr_freed;
	}

	int kbase_mem_pool_grow(struct kbase_mem_pool *pool,
	size_t nr_to_grow)
	{
	struct page *p;
	size_t i;

	kbase_mem_pool_lock(pool);

	pool->dont_reclaim = true;
	for (i = 0; i < nr_to_grow; i++) {
	if (pool->dying) {
	pool->dont_reclaim = false;
	kbase_mem_pool_shrink_locked(pool, nr_to_grow);
	kbase_mem_pool_unlock(pool);

	return -ENOMEM;
	}
	kbase_mem_pool_unlock(pool);

	p = kbase_mem_alloc_page(pool);
	if (!p) {
	kbase_mem_pool_lock(pool);
	pool->dont_reclaim = false;
	kbase_mem_pool_unlock(pool);

	return -ENOMEM;
	}

	kbase_mem_pool_lock(pool);
	kbase_mem_pool_add_locked(pool, p);
	}
	pool->dont_reclaim = false;
	kbase_mem_pool_unlock(pool);

	return 0;
	}

	void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size)
	{
	size_t cur_size;
	int err = 0;

	cur_size = kbase_mem_pool_size(pool);

	if (new_size > pool->max_size)
	new_size = pool->max_size;

	if (new_size < cur_size)
	kbase_mem_pool_shrink(pool, cur_size - new_size);
	else if (new_size > cur_size)
	err = kbase_mem_pool_grow(pool, new_size - cur_size);

	if (err) {
	size_t grown_size = kbase_mem_pool_size(pool);

	dev_warn(pool->kbdev->dev,
	"Mem pool not grown to the required size of %zu bytes, grown for additional %zu bytes instead!\n",
	(new_size - cur_size), (grown_size - cur_size));
	}
	}

	void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size)
	{
	size_t cur_size;
	size_t nr_to_shrink;

	kbase_mem_pool_lock(pool);

	pool->max_size = max_size;

	cur_size = kbase_mem_pool_size(pool);
	if (max_size < cur_size) {
	nr_to_shrink = cur_size - max_size;
	kbase_mem_pool_shrink_locked(pool, nr_to_shrink);
	}

	kbase_mem_pool_unlock(pool);
	}


	static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s,
	struct shrink_control *sc)
	{
	struct kbase_mem_pool *pool;
	size_t pool_size;

	pool = container_of(s, struct kbase_mem_pool, reclaim);

	kbase_mem_pool_lock(pool);
	if (pool->dont_reclaim && !pool->dying) {
	kbase_mem_pool_unlock(pool);
	return 0;
	}
	pool_size = kbase_mem_pool_size(pool);
	kbase_mem_pool_unlock(pool);

	return pool_size;
	}

	static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s,
	struct shrink_control *sc)
	{
	struct kbase_mem_pool *pool;
	unsigned long freed;

	pool = container_of(s, struct kbase_mem_pool, reclaim);

	kbase_mem_pool_lock(pool);
	if (pool->dont_reclaim && !pool->dying) {
	kbase_mem_pool_unlock(pool);
	return 0;
	}

	pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan);

	freed = kbase_mem_pool_shrink_locked(pool, sc->nr_to_scan);

	kbase_mem_pool_unlock(pool);

	pool_dbg(pool, "reclaim freed %ld pages\n", freed);

	return freed;
	}

	#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
	static int kbase_mem_pool_reclaim_shrink(struct shrinker *s,
	struct shrink_control *sc)
	{
	if (sc->nr_to_scan == 0)
	return kbase_mem_pool_reclaim_count_objects(s, sc);

	return kbase_mem_pool_reclaim_scan_objects(s, sc);
	}
	#endif

	int kbase_mem_pool_init(struct kbase_mem_pool *pool,
	size_t max_size,
	size_t order,
	struct kbase_device *kbdev,
	struct kbase_mem_pool *next_pool)
	{
	pool->cur_size = 0;
	pool->max_size = max_size;
	pool->order = order;
	pool->kbdev = kbdev;
	pool->next_pool = next_pool;
	pool->dying = false;

	spin_lock_init(&pool->pool_lock);
	INIT_LIST_HEAD(&pool->page_list);

	/* Register shrinker */
	#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
	pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink;
	#else
	pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects;
	pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects;
	#endif
	pool->reclaim.seeks = DEFAULT_SEEKS;
	/* Kernel versions prior to 3.1 :
	* struct shrinker does not define batch */
	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0)
	pool->reclaim.batch = 0;
	#endif
	register_shrinker(&pool->reclaim);

	pool_dbg(pool, "initialized\n");

	return 0;
	}

	void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool)
	{
	kbase_mem_pool_lock(pool);
	pool->dying = true;
	kbase_mem_pool_unlock(pool);
	}

	void kbase_mem_pool_term(struct kbase_mem_pool *pool)
	{
	struct kbase_mem_pool *next_pool = pool->next_pool;
	struct page *p;
	size_t nr_to_spill = 0;
	LIST_HEAD(spill_list);
	int i;

	pool_dbg(pool, "terminate()\n");

	unregister_shrinker(&pool->reclaim);

	kbase_mem_pool_lock(pool);
	pool->max_size = 0;

	if (next_pool && !kbase_mem_pool_is_full(next_pool)) {
	/* Spill to next pool (may overspill) */
	nr_to_spill = kbase_mem_pool_capacity(next_pool);
	nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill);

	/* Zero pages first without holding the next_pool lock */
	for (i = 0; i < nr_to_spill; i++) {
	p = kbase_mem_pool_remove_locked(pool);
	kbase_mem_pool_zero_page(pool, p);
	list_add(&p->lru, &spill_list);
	}
	}

	while (!kbase_mem_pool_is_empty(pool)) {
	/* Free remaining pages to kernel */
	p = kbase_mem_pool_remove_locked(pool);
	kbase_mem_pool_free_page(pool, p);
	}

	kbase_mem_pool_unlock(pool);

	if (next_pool && nr_to_spill) {
	/* Add new page list to next_pool */
	kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill);

	pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill);
	}

	pool_dbg(pool, "terminated\n");
	}

	struct page kbase_mem_pool_alloc(struct kbase_mem_pool pool)
	{
	struct page *p;

	do {
	pool_dbg(pool, "alloc()\n");
	p = kbase_mem_pool_remove(pool);

	if (p)
	return p;

	pool = pool->next_pool;
	} while (pool);

	return NULL;
	}

	struct page kbase_mem_pool_alloc_locked(struct kbase_mem_pool pool)
	{
	struct page *p;

	lockdep_assert_held(&pool->pool_lock);

	pool_dbg(pool, "alloc_locked()\n");
	p = kbase_mem_pool_remove_locked(pool);

	if (p)
	return p;

	return NULL;
	}

	void kbase_mem_pool_free(struct kbase_mem_pool pool, struct page p,
	bool dirty)
	{
	struct kbase_mem_pool *next_pool = pool->next_pool;

	pool_dbg(pool, "free()\n");

	if (!kbase_mem_pool_is_full(pool)) {
	/* Add to our own pool */
	if (dirty)
	kbase_mem_pool_sync_page(pool, p);

	kbase_mem_pool_add(pool, p);
	} else if (next_pool && !kbase_mem_pool_is_full(next_pool)) {
	/* Spill to next pool */
	kbase_mem_pool_spill(next_pool, p);
	} else {
	/* Free page */
	kbase_mem_pool_free_page(pool, p);
	}
	}

	void kbase_mem_pool_free_locked(struct kbase_mem_pool pool, struct page p,
	bool dirty)
	{
	pool_dbg(pool, "free_locked()\n");

	lockdep_assert_held(&pool->pool_lock);

	if (!kbase_mem_pool_is_full(pool)) {
	/* Add to our own pool */
	if (dirty)
	kbase_mem_pool_sync_page(pool, p);

	kbase_mem_pool_add_locked(pool, p);
	} else {
	/* Free page */
	kbase_mem_pool_free_page(pool, p);
	}
	}

	int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages,
	struct tagged_addr *pages, bool partial_allowed)
	{
	struct page *p;
	size_t nr_from_pool;
	size_t i = 0;
	int err = -ENOMEM;
	size_t nr_pages_internal;

	nr_pages_internal = nr_4k_pages / (1u << (pool->order));

	if (nr_pages_internal * (1u << pool->order) != nr_4k_pages)
	return -EINVAL;

	pool_dbg(pool, "alloc_pages(4k=%zu):\n", nr_4k_pages);
	pool_dbg(pool, "alloc_pages(internal=%zu):\n", nr_pages_internal);

	/* Get pages from this pool */
	kbase_mem_pool_lock(pool);
	nr_from_pool = min(nr_pages_internal, kbase_mem_pool_size(pool));
	while (nr_from_pool--) {
	int j;
	p = kbase_mem_pool_remove_locked(pool);
	if (pool->order) {
	pages[i++] = as_tagged_tag(page_to_phys(p),
	HUGE_HEAD \| HUGE_PAGE);
	for (j = 1; j < (1u << pool->order); j++)
	pages[i++] = as_tagged_tag(page_to_phys(p) +
	PAGE_SIZE * j,
	HUGE_PAGE);
	} else {
	pages[i++] = as_tagged(page_to_phys(p));
	}
	}
	kbase_mem_pool_unlock(pool);

	if (i != nr_4k_pages && pool->next_pool) {
	/* Allocate via next pool */
	err = kbase_mem_pool_alloc_pages(pool->next_pool,
	nr_4k_pages - i, pages + i, partial_allowed);

	if (err < 0)
	goto err_rollback;

	i += err;
	} else {
	/* Get any remaining pages from kernel */
	while (i != nr_4k_pages) {
	p = kbase_mem_alloc_page(pool);
	if (!p) {
	if (partial_allowed)
	goto done;
	else
	goto err_rollback;
	}

	if (pool->order) {
	int j;

	pages[i++] = as_tagged_tag(page_to_phys(p),
	HUGE_PAGE \|
	HUGE_HEAD);
	for (j = 1; j < (1u << pool->order); j++) {
	phys_addr_t phys;

	phys = page_to_phys(p) + PAGE_SIZE * j;
	pages[i++] = as_tagged_tag(phys,
	HUGE_PAGE);
	}
	} else {
	pages[i++] = as_tagged(page_to_phys(p));
	}
	}
	}

	done:
	pool_dbg(pool, "alloc_pages(%zu) done\n", i);
	return i;

	err_rollback:
	kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED);
	return err;
	}

	int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool,
	size_t nr_4k_pages, struct tagged_addr *pages)
	{
	struct page *p;
	size_t i;
	size_t nr_pages_internal;

	lockdep_assert_held(&pool->pool_lock);

	nr_pages_internal = nr_4k_pages / (1u << (pool->order));

	if (nr_pages_internal * (1u << pool->order) != nr_4k_pages)
	return -EINVAL;

	pool_dbg(pool, "alloc_pages_locked(4k=%zu):\n", nr_4k_pages);
	pool_dbg(pool, "alloc_pages_locked(internal=%zu):\n",
	nr_pages_internal);

	if (kbase_mem_pool_size(pool) < nr_pages_internal) {
	pool_dbg(pool, "Failed alloc\n");
	return -ENOMEM;
	}

	for (i = 0; i < nr_pages_internal; i++) {
	int j;

	p = kbase_mem_pool_remove_locked(pool);
	if (pool->order) {
	*pages++ = as_tagged_tag(page_to_phys(p),
	HUGE_HEAD \| HUGE_PAGE);
	for (j = 1; j < (1u << pool->order); j++) {
	*pages++ = as_tagged_tag(page_to_phys(p) +
	PAGE_SIZE * j,
	HUGE_PAGE);
	}
	} else {
	*pages++ = as_tagged(page_to_phys(p));
	}
	}

	return nr_4k_pages;
	}

	static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool,
	size_t nr_pages, struct tagged_addr *pages,
	bool zero, bool sync)
	{
	struct page *p;
	size_t nr_to_pool = 0;
	LIST_HEAD(new_page_list);
	size_t i;

	if (!nr_pages)
	return;

	pool_dbg(pool, "add_array(%zu, zero=%d, sync=%d):\n",
	nr_pages, zero, sync);

	/* Zero/sync pages first without holding the pool lock */
	for (i = 0; i < nr_pages; i++) {
	if (unlikely(!as_phys_addr_t(pages[i])))
	continue;

	if (is_huge_head(pages[i]) \|\| !is_huge(pages[i])) {
	p = phys_to_page(as_phys_addr_t(pages[i]));
	if (zero)
	kbase_mem_pool_zero_page(pool, p);
	else if (sync)
	kbase_mem_pool_sync_page(pool, p);

	list_add(&p->lru, &new_page_list);
	nr_to_pool++;
	}
	pages[i] = as_tagged(0);
	}

	/* Add new page list to pool */
	kbase_mem_pool_add_list(pool, &new_page_list, nr_to_pool);

	pool_dbg(pool, "add_array(%zu) added %zu pages\n",
	nr_pages, nr_to_pool);
	}

	static void kbase_mem_pool_add_array_locked(struct kbase_mem_pool *pool,
	size_t nr_pages, struct tagged_addr *pages,
	bool zero, bool sync)
	{
	struct page *p;
	size_t nr_to_pool = 0;
	LIST_HEAD(new_page_list);
	size_t i;

	lockdep_assert_held(&pool->pool_lock);

	if (!nr_pages)
	return;

	pool_dbg(pool, "add_array_locked(%zu, zero=%d, sync=%d):\n",
	nr_pages, zero, sync);

	/* Zero/sync pages first */
	for (i = 0; i < nr_pages; i++) {
	if (unlikely(!as_phys_addr_t(pages[i])))
	continue;

	if (is_huge_head(pages[i]) \|\| !is_huge(pages[i])) {
	p = phys_to_page(as_phys_addr_t(pages[i]));
	if (zero)
	kbase_mem_pool_zero_page(pool, p);
	else if (sync)
	kbase_mem_pool_sync_page(pool, p);

	list_add(&p->lru, &new_page_list);
	nr_to_pool++;
	}
	pages[i] = as_tagged(0);
	}

	/* Add new page list to pool */
	kbase_mem_pool_add_list_locked(pool, &new_page_list, nr_to_pool);

	pool_dbg(pool, "add_array_locked(%zu) added %zu pages\n",
	nr_pages, nr_to_pool);
	}

	void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages,
	struct tagged_addr *pages, bool dirty, bool reclaimed)
	{
	struct kbase_mem_pool *next_pool = pool->next_pool;
	struct page *p;
	size_t nr_to_pool;
	LIST_HEAD(to_pool_list);
	size_t i = 0;

	pool_dbg(pool, "free_pages(%zu):\n", nr_pages);

	if (!reclaimed) {
	/* Add to this pool */
	nr_to_pool = kbase_mem_pool_capacity(pool);
	nr_to_pool = min(nr_pages, nr_to_pool);

	kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty);

	i += nr_to_pool;

	if (i != nr_pages && next_pool) {
	/* Spill to next pool (may overspill) */
	nr_to_pool = kbase_mem_pool_capacity(next_pool);
	nr_to_pool = min(nr_pages - i, nr_to_pool);

	kbase_mem_pool_add_array(next_pool, nr_to_pool,
	pages + i, true, dirty);
	i += nr_to_pool;
	}
	}

	/* Free any remaining pages to kernel */
	for (; i < nr_pages; i++) {
	if (unlikely(!as_phys_addr_t(pages[i])))
	continue;

	if (is_huge(pages[i]) && !is_huge_head(pages[i])) {
	pages[i] = as_tagged(0);
	continue;
	}

	p = phys_to_page(as_phys_addr_t(pages[i]));

	kbase_mem_pool_free_page(pool, p);
	pages[i] = as_tagged(0);
	}

	pool_dbg(pool, "free_pages(%zu) done\n", nr_pages);
	}


	void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool,
	size_t nr_pages, struct tagged_addr *pages, bool dirty,
	bool reclaimed)
	{
	struct page *p;
	size_t nr_to_pool;
	LIST_HEAD(to_pool_list);
	size_t i = 0;

	lockdep_assert_held(&pool->pool_lock);

	pool_dbg(pool, "free_pages_locked(%zu):\n", nr_pages);

	if (!reclaimed) {
	/* Add to this pool */
	nr_to_pool = kbase_mem_pool_capacity(pool);
	nr_to_pool = min(nr_pages, nr_to_pool);

	kbase_mem_pool_add_array_locked(pool, nr_pages, pages, false,
	dirty);

	i += nr_to_pool;
	}

	/* Free any remaining pages to kernel */
	for (; i < nr_pages; i++) {
	if (unlikely(!as_phys_addr_t(pages[i])))
	continue;

	if (is_huge(pages[i]) && !is_huge_head(pages[i])) {
	pages[i] = as_tagged(0);
	continue;
	}

	p = phys_to_page(as_phys_addr_t(pages[i]));

	kbase_mem_pool_free_page(pool, p);
	pages[i] = as_tagged(0);
	}

	pool_dbg(pool, "free_pages_locked(%zu) done\n", nr_pages);
	}