blob: c1a8e9ceb2faef0bef0d2145a9b21c219c2b9942 [file] [log] [blame]
/*
*
* (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
*/
/*
* Job Scheduler Implementation
*/
#include <mali_kbase.h>
#include <mali_kbase_js.h>
#if defined(CONFIG_MALI_GATOR_SUPPORT)
#include <mali_kbase_gator.h>
#endif
#include <mali_kbase_tlstream.h>
#include <mali_kbase_hw.h>
#include <mali_kbase_defs.h>
#include <mali_kbase_config_defaults.h>
#include "mali_kbase_jm.h"
#include "mali_kbase_hwaccess_jm.h"
/*
* Private types
*/
/* Bitpattern indicating the result of releasing a context */
enum {
/* The context was descheduled - caller should try scheduling in a new
* one to keep the runpool full */
KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0),
/* Ctx attributes were changed - caller should try scheduling all
* contexts */
KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1)
};
typedef u32 kbasep_js_release_result;
const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = {
KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */
KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */
KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */
};
const base_jd_prio
kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = {
BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */
BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */
BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */
};
/*
* Private function prototypes
*/
static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
struct kbase_device *kbdev, struct kbase_context *kctx,
struct kbasep_js_atom_retained_state *katom_retained_state);
static int kbase_js_get_slot(struct kbase_device *kbdev,
struct kbase_jd_atom *katom);
static void kbase_js_foreach_ctx_job(struct kbase_context *kctx,
kbasep_js_policy_ctx_job_cb callback);
/* Helper for trace subcodes */
#if KBASE_TRACE_ENABLE
static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
int as_nr;
int refcnt = 0;
js_devdata = &kbdev->js_data;
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
as_nr = kctx->as_nr;
if (as_nr != KBASEP_AS_NR_INVALID) {
struct kbasep_js_per_as_data *js_per_as_data;
js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
refcnt = js_per_as_data->as_busy_refcount;
}
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
return refcnt;
}
static int kbasep_js_trace_get_refcnt_nolock(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
int as_nr;
int refcnt = 0;
js_devdata = &kbdev->js_data;
as_nr = kctx->as_nr;
if (as_nr != KBASEP_AS_NR_INVALID) {
struct kbasep_js_per_as_data *js_per_as_data;
js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
refcnt = js_per_as_data->as_busy_refcount;
}
return refcnt;
}
#else /* KBASE_TRACE_ENABLE */
static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
CSTD_UNUSED(kbdev);
CSTD_UNUSED(kctx);
return 0;
}
static int kbasep_js_trace_get_refcnt_nolock(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
CSTD_UNUSED(kbdev);
CSTD_UNUSED(kctx);
return 0;
}
#endif /* KBASE_TRACE_ENABLE */
/*
* Private types
*/
enum {
JS_DEVDATA_INIT_NONE = 0,
JS_DEVDATA_INIT_CONSTANTS = (1 << 0),
JS_DEVDATA_INIT_POLICY = (1 << 1),
JS_DEVDATA_INIT_ALL = ((1 << 2) - 1)
};
enum {
JS_KCTX_INIT_NONE = 0,
JS_KCTX_INIT_CONSTANTS = (1 << 0),
JS_KCTX_INIT_POLICY = (1 << 1),
JS_KCTX_INIT_ALL = ((1 << 2) - 1)
};
/*
* Private functions
*/
/**
* core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements
* @features: JSn_FEATURE register value
*
* Given a JSn_FEATURE register value returns the core requirements that match
*
* Return: Core requirement bit mask
*/
static base_jd_core_req core_reqs_from_jsn_features(u16 features)
{
base_jd_core_req core_req = 0u;
if ((features & JS_FEATURE_SET_VALUE_JOB) != 0)
core_req |= BASE_JD_REQ_V;
if ((features & JS_FEATURE_CACHE_FLUSH_JOB) != 0)
core_req |= BASE_JD_REQ_CF;
if ((features & JS_FEATURE_COMPUTE_JOB) != 0)
core_req |= BASE_JD_REQ_CS;
if ((features & JS_FEATURE_TILER_JOB) != 0)
core_req |= BASE_JD_REQ_T;
if ((features & JS_FEATURE_FRAGMENT_JOB) != 0)
core_req |= BASE_JD_REQ_FS;
return core_req;
}
static void kbase_js_sync_timers(struct kbase_device *kbdev)
{
mutex_lock(&kbdev->js_data.runpool_mutex);
kbase_backend_ctx_count_changed(kbdev);
mutex_unlock(&kbdev->js_data.runpool_mutex);
}
/* Hold the kbasep_js_device_data::runpool_irq::lock for this */
bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_per_as_data *js_per_as_data;
bool result = false;
int as_nr;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
js_devdata = &kbdev->js_data;
as_nr = kctx->as_nr;
if (as_nr != KBASEP_AS_NR_INVALID) {
int new_refcnt;
KBASE_DEBUG_ASSERT(as_nr >= 0);
js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
KBASE_DEBUG_ASSERT(js_per_as_data->kctx != NULL);
new_refcnt = ++(js_per_as_data->as_busy_refcount);
KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx,
NULL, 0u, new_refcnt);
result = true;
}
return result;
}
/* Helper macros to access and modify jsctx_queue.indicies */
#define JSCTX_GET(offset, var, mask) \
((var >> offset) & mask)
/* This wraps around to correct integer size automatically. */
#define JSCTX_SET(var, offset, value, mask) \
(var = ((var & ~(mask << offset)) /*Clear old bits */ \
| (((value) & mask) << offset))) /* Set (after masking) new bits */
#define JSCTX_GET_WR_IDX(var) \
JSCTX_GET(JSCTX_WR_OFFSET, var, JSCTX_RB_MASK_STORE)
#define JSCTX_GET_RN_IDX(var) \
JSCTX_GET(JSCTX_RN_OFFSET, var, JSCTX_RB_MASK_STORE)
#define JSCTX_GET_RD_IDX(var) \
JSCTX_GET(JSCTX_RD_OFFSET, var, JSCTX_RB_MASK_STORE)
#define JSCTX_GET_IDX_DIFF(lower, upper) \
((upper >= lower) ? (upper - lower) : (upper+JSCTX_RB_SIZE_STORE-lower))
#define JSCTX_SET_WR_IDX(var, value) \
JSCTX_SET(var, JSCTX_WR_OFFSET, value, JSCTX_RB_MASK_STORE)
#define JSCTX_SET_RN_IDX(var, value) \
JSCTX_SET(var, JSCTX_RN_OFFSET, value, JSCTX_RB_MASK_STORE)
#define JSCTX_SET_RD_IDX(var, value) \
JSCTX_SET(var, JSCTX_RD_OFFSET, value, JSCTX_RB_MASK_STORE)
/**
* jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms
* @kctx: Pointer to kbase context with ring buffer.
* @js: Job slot id to check.
* @prio: Priority to check.
*
* Return true if there are no atoms to pull. There may be running atoms in the
* ring buffer even if there are no atoms to pull. It is also possible for the
* ring buffer to be full (with running atoms) when this functions returns
* true.
*
* Return: true if there are no atoms to pull, false otherwise.
*/
static inline bool
jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio)
{
struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js];
unsigned int var = atomic_read(&rb->indicies);
return JSCTX_GET_RD_IDX(var) == JSCTX_GET_WR_IDX(var);
}
/**
* jsctx_rb_none_to_pull(): - Check if all priority ring buffers have no
* pullable atoms
* @kctx: Pointer to kbase context with ring buffer.
* @js: Job slot id to check.
*
* Caller must hold runpool_irq.lock
*
* Return: true if the ring buffers for all priorities have no pullable atoms,
* false otherwise.
*/
static inline bool
jsctx_rb_none_to_pull(struct kbase_context *kctx, int js)
{
int prio;
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) {
if (!jsctx_rb_none_to_pull_prio(kctx, js, prio))
return false;
}
return true;
}
/**
* jsctx_rb_is_full(): - Check if the given ringbuffer is full.
* @queue: Pointer to the queue containing the ringbuffer.
*
* No locks explicitly required, result will always be consistent.
* But depending on usage, the caller should consider jctx.lock,
* for the result to remain correct.
*
* Return: true if the ringbuffer is full, false otherwise.
*/
static inline bool
jsctx_rb_is_full(struct jsctx_queue *queue)
{
unsigned int var = atomic_read(&queue->indicies);
u16 rn_idx = JSCTX_GET_RN_IDX(var);
u16 wr_idx = JSCTX_GET_WR_IDX(var);
return JSCTX_GET_IDX_DIFF(rn_idx, wr_idx) >= JSCTX_RB_SIZE;
}
/**
* jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue.
* @kctx: Pointer to kbase context with the queue.
* @js: Job slot id to iterate.
* @prio: Priority id to iterate.
* @callback: Function pointer to callback.
*
* Iterate over a ring buffer and invoke @callback for each entry in buffer, and
* remove the entry from the buffer.
*
* If entries are added to the ring buffer while this is running those entries
* may, or may not be covered. To ensure that all entries in the buffer have
* been enumerated when this function returns jsctx->lock must be held when
* calling this function.
*
* The HW access lock, js_data.runpool_irq.lock, must always be held when
* calling this function.
*/
static void
jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio,
kbasep_js_policy_ctx_job_cb callback)
{
struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js];
struct kbase_jd_atom *katom;
struct list_head *pos, *q;
unsigned int var = atomic_read(&queue->indicies);
u16 running_idx = JSCTX_GET_RN_IDX(var);
u16 read_idx = JSCTX_GET_RD_IDX(var);
u16 wr_idx = JSCTX_GET_WR_IDX(var);
u16 i;
const u16 count = JSCTX_GET_IDX_DIFF(running_idx, wr_idx);
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
/* There must be no jobs currently in HW access */
WARN_ON(read_idx != JSCTX_GET_RN_IDX(var));
/* Invoke callback on all kbase_jd_atoms in the ring buffer, and
* removes them from the buffer */
for (i = 0; i < count; i++) {
int id = queue->entries[read_idx & JSCTX_RB_MASK].atom_id;
katom = kbase_jd_atom_from_id(kctx, id);
read_idx++;
callback(kctx->kbdev, katom);
}
atomic_set(&queue->indicies, 0);
list_for_each_safe(pos, q, &queue->queue_head) {
struct kbase_jd_atom *entry;
entry = list_entry(pos, struct kbase_jd_atom, queue);
list_del(pos);
callback(kctx->kbdev, entry);
}
}
/**
* jsctx_queue_foreach(): - Execute callback for each entry in every queue
* @kctx: Pointer to kbase context with queue.
* @js: Job slot id to iterate.
* @callback: Function pointer to callback.
*
* Iterate over all the different priorities, and for each call
* jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback
* for each entry, and remove the entry from the queue.
*/
static inline void
jsctx_queue_foreach(struct kbase_context *kctx, int js,
kbasep_js_policy_ctx_job_cb callback)
{
int prio;
for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++)
jsctx_queue_foreach_prio(kctx, js, prio, callback);
}
/**
* jsctx_rb_peek_prio(): - Check buffer and get next atom
* @kctx: Pointer to kbase context with ring buffer.
* @js: Job slot id to check.
* @prio: Priority id to check.
*
* Check the ring buffer for the specified @js and @prio and return a pointer to
* the next atom, unless the ring buffer is empty.
*
* Return: Pointer to next atom in buffer, or NULL if there is no atom.
*/
static inline struct kbase_jd_atom *
jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio)
{
struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js];
int id;
unsigned int var = atomic_read(&rb->indicies);
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
if (JSCTX_GET_RD_IDX(var) == JSCTX_GET_WR_IDX(var))
return NULL;
id = rb->entries[JSCTX_GET_RD_IDX(var) & JSCTX_RB_MASK].atom_id;
return kbase_jd_atom_from_id(kctx, id);
}
/**
* jsctx_rb_peek(): - Check all priority buffers and get next atom
* @kctx: Pointer to kbase context with ring buffer.
* @js: Job slot id to check.
*
* Check the ring buffers for all priorities, starting from
* KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a
* pointer to the next atom, unless all the priority's ring buffers are empty.
*
* Caller must hold the runpool_irq.lock.
*
* Return: Pointer to next atom in buffer, or NULL if there is no atom.
*/
static inline struct kbase_jd_atom *
jsctx_rb_peek(struct kbase_context *kctx, int js)
{
int prio;
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) {
struct kbase_jd_atom *katom;
katom = jsctx_rb_peek_prio(kctx, js, prio);
if (katom)
return katom;
}
return NULL;
}
/**
* jsctx_rb_peek_last(): - Check a ring buffer and get the last atom
* @kctx: Pointer to kbase context with ring buffer.
* @js: Job slot id to check.
* @prio: Priority id to check.
*
* Check the ring buffer for the specified @js and @prio and return a
* pointer to the last atom, unless all the priority's ring buffers are empty.
*
* The last atom is the atom that was added using jsctx_rb_add() most recently.
*
* Return: Pointer to last atom in buffer, or NULL if there is no atom.
*/
static inline struct kbase_jd_atom *
jsctx_rb_peek_last(struct kbase_context *kctx, int js, int prio)
{
struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js];
unsigned int var = atomic_read(&rb->indicies);
int id;
lockdep_assert_held(&kctx->jctx.lock);
if (!list_empty(&rb->queue_head)) {
return list_entry(rb->queue_head.prev,
struct kbase_jd_atom, queue);
}
if (JSCTX_GET_RN_IDX(var) == JSCTX_GET_WR_IDX(var))
return NULL;
id = rb->entries[(JSCTX_GET_WR_IDX(var) - 1) & JSCTX_RB_MASK].atom_id;
return kbase_jd_atom_from_id(kctx, id);
}
/**
* jsctx_rb_pull(): - Mark atom in list as running
* @kctx: Pointer to kbase context with ring buffer.
* @katom: Pointer to katom to pull.
*
* Mark an atom previously obtained from jsctx_rb_peek() as running.
*
* @katom must currently be at the head of the ring buffer.
*/
static inline void
jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
int prio = katom->sched_priority;
int js = katom->slot_nr;
unsigned int oldvar, var;
struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js];
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
/* Atoms must be pulled in the correct order. */
WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio));
do {
u16 rd_idx;
oldvar = atomic_read(&rb->indicies);
var = oldvar;
rd_idx = JSCTX_GET_RD_IDX(var);
JSCTX_SET_RD_IDX(var, rd_idx+1);
} while (atomic_cmpxchg(&rb->indicies, oldvar, var) != oldvar);
}
/**
* jsctx_rb_unpull(): - Undo marking of atom in list as running
* @kctx: Pointer to kbase context with ring buffer.
* @katom: Pointer to katom to unpull.
*
* Undo jsctx_rb_pull() and put @katom back in the queue.
*
* jsctx_rb_unpull() must be called on atoms in the same order the atoms were
* pulled.
*/
static inline void
jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
int prio = katom->sched_priority;
int js = katom->slot_nr;
unsigned int oldvar, var;
struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js];
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
do {
u16 rd_idx;
oldvar = atomic_read(&rb->indicies);
var = oldvar;
rd_idx = JSCTX_GET_RD_IDX(var)-1;
/* Atoms must be unpulled in correct order. */
WARN_ON(rb->entries[rd_idx & JSCTX_RB_MASK].atom_id !=
kbase_jd_atom_id(kctx, katom));
JSCTX_SET_RD_IDX(var, rd_idx);
} while (atomic_cmpxchg(&rb->indicies, oldvar, var) != oldvar);
}
/**
* jsctx_rb_add(): - Add atom to ring buffer
* @kctx: Pointer to kbase context with ring buffer.
* @katom: Pointer to katom to add.
*
* Add @katom to the ring buffer determined by the atom's priority and job slot
* number.
*
* If the ring buffer is full -EBUSY will be returned.
*
* Return: On success 0 is returned, on failure a negative error code.
*/
static int
jsctx_rb_add_atom(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
int prio = katom->sched_priority;
int js = katom->slot_nr;
struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js];
unsigned int oldvar, var;
u16 wr_idx, running_idx, count;
lockdep_assert_held(&kctx->jctx.lock);
oldvar = atomic_read(&rb->indicies);
var = oldvar;
running_idx = JSCTX_GET_RN_IDX(var);
wr_idx = JSCTX_GET_WR_IDX(var);
count = JSCTX_GET_IDX_DIFF(running_idx, wr_idx);
/* Check if the ring buffer is full */
if (count >= JSCTX_RB_SIZE)
return -EBUSY;
rb->entries[wr_idx & JSCTX_RB_MASK].atom_id =
kbase_jd_atom_id(kctx, katom);
wr_idx++;
JSCTX_SET_WR_IDX(var, wr_idx);
while (atomic_cmpxchg(&rb->indicies, oldvar, var) != oldvar) {
oldvar = atomic_read(&rb->indicies);
var = oldvar;
wr_idx = JSCTX_GET_WR_IDX(var)+1;
JSCTX_SET_WR_IDX(var, wr_idx);
}
return 0;
}
/**
* jsctx_rb_remove(): - Remove atom from ring buffer
* @kctx: Pointer to kbase context with ring buffer.
* @katom: Pointer to katom to remove.
*
* Remove @katom from the ring buffer.
*
* @katom must have been pulled from the buffer earlier by jsctx_rb_pull(), and
* atoms must be removed in the same order they were pulled from the ring
* buffer.
*/
static inline void
jsctx_rb_remove(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
int prio = katom->sched_priority;
int js = katom->slot_nr;
unsigned int oldvar, var;
struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js];
lockdep_assert_held(&kctx->jctx.lock);
do {
unsigned int rn_idx;
oldvar = atomic_read(&rb->indicies);
var = oldvar;
rn_idx = JSCTX_GET_RN_IDX(var);
JSCTX_SET_RN_IDX(var, rn_idx+1);
} while (atomic_cmpxchg(&rb->indicies, oldvar, var) != oldvar);
}
static void
jsctx_ll_add(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
int prio = katom->sched_priority;
int js = katom->slot_nr;
struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js];
lockdep_assert_held(&kctx->jctx.lock);
list_add_tail(&katom->queue, &queue->queue_head);
}
static bool kbase_js_ctx_pullable(struct kbase_context *kctx,
int js,
bool is_scheduled);
static bool kbase_js_ctx_list_add_pullable(struct kbase_device *kbdev,
struct kbase_context *kctx,
int js);
static bool kbase_js_ctx_list_add_unpullable(struct kbase_device *kbdev,
struct kbase_context *kctx,
int js);
void
jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js)
{
unsigned long flags;
struct list_head *pos, *q;
struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js];
bool flushed_any = false;
struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data;
bool enqueue_required = false;
lockdep_assert_held(&kctx->jctx.lock);
/* Early out for common case */
if (list_empty(&queue->queue_head) || jsctx_rb_is_full(queue))
return;
mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
spin_lock_irqsave(&kctx->kbdev->js_data.runpool_irq.lock, flags);
/* If slot will transition from unpullable to pullable then add to
* pullable list */
if (jsctx_rb_none_to_pull(kctx, js))
enqueue_required = true;
else
enqueue_required = false;
list_for_each_safe(pos, q, &queue->queue_head) {
struct kbase_jd_atom *katom;
katom = list_entry(pos, struct kbase_jd_atom, queue);
KBASE_DEBUG_ASSERT(katom);
if (jsctx_rb_add_atom(kctx, katom))
break;
katom->atom_flags &= ~KBASE_KATOM_FLAG_JSCTX_IN_LL;
katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED;
flushed_any = true;
list_del(pos);
}
if (flushed_any) {
bool timer_sync = false;
if (enqueue_required) {
if (kbase_js_ctx_pullable(kctx, js, false))
timer_sync = kbase_js_ctx_list_add_pullable(
kctx->kbdev, kctx, js);
else
timer_sync = kbase_js_ctx_list_add_unpullable(
kctx->kbdev, kctx, js);
/* If this context is active and the atom is the first
* on its slot, kick the job manager to attempt to
* fast-start the atom */
if (kctx == kctx->kbdev->hwaccess.active_kctx)
kbase_jm_try_kick(kctx->kbdev, 1 << js);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
flags);
if (timer_sync)
kbase_backend_ctx_count_changed(kctx->kbdev);
} else {
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
flags);
}
} else {
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
}
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
}
/*
* Functions private to KBase ('Protected' functions)
*/
int kbasep_js_devdata_init(struct kbase_device * const kbdev)
{
struct kbasep_js_device_data *jsdd;
int err;
int i;
u16 as_present;
KBASE_DEBUG_ASSERT(kbdev != NULL);
jsdd = &kbdev->js_data;
KBASE_DEBUG_ASSERT(jsdd->init_status == JS_DEVDATA_INIT_NONE);
/* These two must be recalculated if nr_hw_address_spaces changes
* (e.g. for HW workarounds) */
as_present = (1U << kbdev->nr_hw_address_spaces) - 1;
kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
bool use_workaround;
use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE;
if (use_workaround) {
dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only");
kbdev->nr_user_address_spaces = 1;
}
}
#ifdef CONFIG_MALI_DEBUG
/* Soft-stop will be disabled on a single context by default unless
* softstop_always is set */
jsdd->softstop_always = false;
#endif /* CONFIG_MALI_DEBUG */
jsdd->nr_all_contexts_running = 0;
jsdd->nr_user_contexts_running = 0;
jsdd->nr_contexts_pullable = 0;
atomic_set(&jsdd->nr_contexts_runnable, 0);
/* All ASs initially free */
jsdd->as_free = as_present;
/* No ctx allowed to submit */
jsdd->runpool_irq.submit_allowed = 0u;
memset(jsdd->runpool_irq.ctx_attr_ref_count, 0,
sizeof(jsdd->runpool_irq.ctx_attr_ref_count));
memset(jsdd->runpool_irq.slot_affinities, 0,
sizeof(jsdd->runpool_irq.slot_affinities));
memset(jsdd->runpool_irq.slot_affinity_refcount, 0,
sizeof(jsdd->runpool_irq.slot_affinity_refcount));
INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list);
/* Config attributes */
jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS;
jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS;
jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS_8408;
else
jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS;
jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL;
jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS_8408;
else
jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS;
jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL;
jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING;
jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS;
jsdd->cfs_ctx_runtime_init_slices =
DEFAULT_JS_CFS_CTX_RUNTIME_INIT_SLICES;
jsdd->cfs_ctx_runtime_min_slices =
DEFAULT_JS_CFS_CTX_RUNTIME_MIN_SLICES;
atomic_set(&jsdd->soft_event_timeout_ms, DEFAULT_JS_SOFT_EVENT_TIMEOUT);
dev_dbg(kbdev->dev, "JS Config Attribs: ");
dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u",
jsdd->scheduling_period_ns);
dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u",
jsdd->soft_stop_ticks);
dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u",
jsdd->soft_stop_ticks_cl);
dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u",
jsdd->hard_stop_ticks_ss);
dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u",
jsdd->hard_stop_ticks_cl);
dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u",
jsdd->hard_stop_ticks_dumping);
dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u",
jsdd->gpu_reset_ticks_ss);
dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u",
jsdd->gpu_reset_ticks_cl);
dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u",
jsdd->gpu_reset_ticks_dumping);
dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u",
jsdd->ctx_timeslice_ns);
dev_dbg(kbdev->dev, "\tcfs_ctx_runtime_init_slices:%u",
jsdd->cfs_ctx_runtime_init_slices);
dev_dbg(kbdev->dev, "\tcfs_ctx_runtime_min_slices:%u",
jsdd->cfs_ctx_runtime_min_slices);
dev_dbg(kbdev->dev, "\tsoft_event_timeout:%i",
atomic_read(&jsdd->soft_event_timeout_ms));
if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss &&
jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss &&
jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping &&
jsdd->hard_stop_ticks_dumping <
jsdd->gpu_reset_ticks_dumping)) {
dev_err(kbdev->dev, "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n");
return -EINVAL;
}
#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS
dev_dbg(kbdev->dev, "Job Scheduling Policy Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.",
jsdd->soft_stop_ticks,
jsdd->scheduling_period_ns);
#endif
#if KBASE_DISABLE_SCHEDULING_HARD_STOPS
dev_dbg(kbdev->dev, "Job Scheduling Policy Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.",
jsdd->hard_stop_ticks_ss,
jsdd->hard_stop_ticks_dumping,
jsdd->scheduling_period_ns);
#endif
#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS
dev_dbg(kbdev->dev, "Note: The JS policy's tick timer (if coded) will still be run, but do nothing.");
#endif
/* setup the number of irq throttle cycles base on given time */
{
int time_us = kbdev->gpu_props.irq_throttle_time_us;
int cycles = kbasep_js_convert_us_to_gpu_ticks_max_freq(kbdev,
time_us);
atomic_set(&kbdev->irq_throttle_cycles, cycles);
}
/* Clear the AS data, including setting NULL pointers */
memset(&jsdd->runpool_irq.per_as_data[0], 0,
sizeof(jsdd->runpool_irq.per_as_data));
for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i)
jsdd->js_reqs[i] = core_reqs_from_jsn_features(
kbdev->gpu_props.props.raw_props.js_features[i]);
jsdd->init_status |= JS_DEVDATA_INIT_CONSTANTS;
/* On error, we could continue on: providing none of the below resources
* rely on the ones above */
mutex_init(&jsdd->runpool_mutex);
mutex_init(&jsdd->queue_mutex);
spin_lock_init(&jsdd->runpool_irq.lock);
sema_init(&jsdd->schedule_sem, 1);
err = kbasep_js_policy_init(kbdev);
if (!err)
jsdd->init_status |= JS_DEVDATA_INIT_POLICY;
for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) {
INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i]);
INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i]);
}
/* On error, do no cleanup; this will be handled by the caller(s), since
* we've designed this resource to be safe to terminate on init-fail */
if (jsdd->init_status != JS_DEVDATA_INIT_ALL)
return -EINVAL;
return 0;
}
void kbasep_js_devdata_halt(struct kbase_device *kbdev)
{
CSTD_UNUSED(kbdev);
}
void kbasep_js_devdata_term(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_devdata;
KBASE_DEBUG_ASSERT(kbdev != NULL);
js_devdata = &kbdev->js_data;
if ((js_devdata->init_status & JS_DEVDATA_INIT_CONSTANTS)) {
s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { 0, };
/* The caller must de-register all contexts before calling this
*/
KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0);
KBASE_DEBUG_ASSERT(memcmp(
js_devdata->runpool_irq.ctx_attr_ref_count,
zero_ctx_attr_ref_count,
sizeof(zero_ctx_attr_ref_count)) == 0);
CSTD_UNUSED(zero_ctx_attr_ref_count);
}
if ((js_devdata->init_status & JS_DEVDATA_INIT_POLICY))
kbasep_js_policy_term(&js_devdata->policy);
js_devdata->init_status = JS_DEVDATA_INIT_NONE;
}
int kbasep_js_kctx_init(struct kbase_context * const kctx)
{
struct kbase_device *kbdev;
struct kbasep_js_kctx_info *js_kctx_info;
int err;
int i, j;
KBASE_DEBUG_ASSERT(kctx != NULL);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(kbdev != NULL);
for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i)
INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]);
js_kctx_info = &kctx->jctx.sched_info;
KBASE_DEBUG_ASSERT(js_kctx_info->init_status == JS_KCTX_INIT_NONE);
js_kctx_info->ctx.nr_jobs = 0;
js_kctx_info->ctx.is_scheduled = false;
js_kctx_info->ctx.is_dying = false;
memset(js_kctx_info->ctx.ctx_attr_ref_count, 0,
sizeof(js_kctx_info->ctx.ctx_attr_ref_count));
/* Initially, the context is disabled from submission until the create
* flags are set */
js_kctx_info->ctx.flags = KBASE_CTX_FLAG_SUBMIT_DISABLED;
js_kctx_info->init_status |= JS_KCTX_INIT_CONSTANTS;
/* On error, we could continue on: providing none of the below resources
* rely on the ones above */
mutex_init(&js_kctx_info->ctx.jsctx_mutex);
init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait);
err = kbasep_js_policy_init_ctx(kbdev, kctx);
if (!err)
js_kctx_info->init_status |= JS_KCTX_INIT_POLICY;
/* On error, do no cleanup; this will be handled by the caller(s), since
* we've designed this resource to be safe to terminate on init-fail */
if (js_kctx_info->init_status != JS_KCTX_INIT_ALL)
return -EINVAL;
for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) {
for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) {
INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].queue_head);
atomic_set(&kctx->jsctx_queue[i][j].indicies, 0);
}
}
return 0;
}
void kbasep_js_kctx_term(struct kbase_context *kctx)
{
struct kbase_device *kbdev;
struct kbasep_js_kctx_info *js_kctx_info;
union kbasep_js_policy *js_policy;
int js;
bool update_ctx_count = false;
KBASE_DEBUG_ASSERT(kctx != NULL);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(kbdev != NULL);
js_policy = &kbdev->js_data.policy;
js_kctx_info = &kctx->jctx.sched_info;
if ((js_kctx_info->init_status & JS_KCTX_INIT_CONSTANTS)) {
/* The caller must de-register all jobs before calling this */
KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0);
}
mutex_lock(&kbdev->js_data.queue_mutex);
mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
for (js = 0; js < kbdev->gpu_props.num_job_slots; js++)
list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
if (kctx->ctx_runnable_ref) {
WARN_ON(atomic_read(&kbdev->js_data.nr_contexts_runnable) <= 0);
atomic_dec(&kbdev->js_data.nr_contexts_runnable);
update_ctx_count = true;
kctx->ctx_runnable_ref = false;
}
mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
mutex_unlock(&kbdev->js_data.queue_mutex);
if ((js_kctx_info->init_status & JS_KCTX_INIT_POLICY))
kbasep_js_policy_term_ctx(js_policy, kctx);
js_kctx_info->init_status = JS_KCTX_INIT_NONE;
if (update_ctx_count) {
mutex_lock(&kbdev->js_data.runpool_mutex);
kbase_backend_ctx_count_changed(kbdev);
mutex_unlock(&kbdev->js_data.runpool_mutex);
}
}
/**
* kbase_js_ctx_list_add_pullable - Add context to the tail of the per-slot
* pullable context queue
* @kbdev: Device pointer
* @kctx: Context to add to queue
* @js: Job slot to use
*
* If the context is on either the pullable or unpullable queues, then it is
* removed before being added to the tail.
*
* This function should be used when queueing a context for the first time, or
* re-queueing a context that has been pulled from.
*
* Caller must hold kbasep_jd_device_data.queue_mutex
*
* Return: true if caller should call kbase_backend_ctx_count_changed()
*/
static bool kbase_js_ctx_list_add_pullable(struct kbase_device *kbdev,
struct kbase_context *kctx,
int js)
{
bool ret = false;
lockdep_assert_held(&kbdev->js_data.queue_mutex);
lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]))
list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js],
&kbdev->js_data.ctx_list_pullable[js]);
if (!kctx->slots_pullable) {
kbdev->js_data.nr_contexts_pullable++;
ret = true;
if (!atomic_read(&kctx->atoms_pulled)) {
WARN_ON(kctx->ctx_runnable_ref);
kctx->ctx_runnable_ref = true;
atomic_inc(&kbdev->js_data.nr_contexts_runnable);
}
}
kctx->slots_pullable |= (1 << js);
return ret;
}
/**
* kbase_js_ctx_list_add_pullable_head - Add context to the head of the
* per-slot pullable context queue
* @kbdev: Device pointer
* @kctx: Context to add to queue
* @js: Job slot to use
*
* If the context is on either the pullable or unpullable queues, then it is
* removed before being added to the head.
*
* This function should be used when a context has been scheduled, but no jobs
* can currently be pulled from it.
*
* Caller must hold kbasep_jd_device_data.queue_mutex
*
* Return: true if caller should call kbase_backend_ctx_count_changed()
*/
static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev,
struct kbase_context *kctx,
int js)
{
bool ret = false;
lockdep_assert_held(&kbdev->js_data.queue_mutex);
lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]))
list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js],
&kbdev->js_data.ctx_list_pullable[js]);
if (!kctx->slots_pullable) {
kbdev->js_data.nr_contexts_pullable++;
ret = true;
if (!atomic_read(&kctx->atoms_pulled)) {
WARN_ON(kctx->ctx_runnable_ref);
kctx->ctx_runnable_ref = true;
atomic_inc(&kbdev->js_data.nr_contexts_runnable);
}
}
kctx->slots_pullable |= (1 << js);
return ret;
}
/**
* kbase_js_ctx_list_add_unpullable - Add context to the tail of the per-slot
* unpullable context queue
* @kbdev: Device pointer
* @kctx: Context to add to queue
* @js: Job slot to use
*
* The context must already be on the per-slot pullable queue. It will be
* removed from the pullable queue before being added to the unpullable queue.
*
* This function should be used when a context has been pulled from, and there
* are no jobs remaining on the specified slot.
*
* Caller must hold kbasep_jd_device_data.queue_mutex
*
* Return: true if caller should call kbase_backend_ctx_count_changed()
*/
static bool kbase_js_ctx_list_add_unpullable(struct kbase_device *kbdev,
struct kbase_context *kctx,
int js)
{
bool ret = false;
lockdep_assert_held(&kbdev->js_data.queue_mutex);
lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js],
&kbdev->js_data.ctx_list_unpullable[js]);
if (kctx->slots_pullable == (1 << js)) {
kbdev->js_data.nr_contexts_pullable--;
ret = true;
if (!atomic_read(&kctx->atoms_pulled)) {
WARN_ON(!kctx->ctx_runnable_ref);
kctx->ctx_runnable_ref = false;
atomic_dec(&kbdev->js_data.nr_contexts_runnable);
}
}
kctx->slots_pullable &= ~(1 << js);
return ret;
}
/**
* kbase_js_ctx_list_remove - Remove context from the per-slot pullable or
* unpullable context queues
* @kbdev: Device pointer
* @kctx: Context to remove from queue
* @js: Job slot to use
*
* The context must already be on one of the queues.
*
* This function should be used when a context has no jobs on the GPU, and no
* jobs remaining for the specified slot.
*
* Caller must hold kbasep_jd_device_data.queue_mutex
*
* Return: true if caller should call kbase_backend_ctx_count_changed()
*/
static bool kbase_js_ctx_list_remove(struct kbase_device *kbdev,
struct kbase_context *kctx,
int js)
{
bool ret = false;
lockdep_assert_held(&kbdev->js_data.queue_mutex);
lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]));
list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
if (kctx->slots_pullable == (1 << js)) {
kbdev->js_data.nr_contexts_pullable--;
ret = true;
if (!atomic_read(&kctx->atoms_pulled)) {
WARN_ON(!kctx->ctx_runnable_ref);
kctx->ctx_runnable_ref = false;
atomic_dec(&kbdev->js_data.nr_contexts_runnable);
}
}
kctx->slots_pullable &= ~(1 << js);
return ret;
}
/**
* kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable
* queue.
* @kbdev: Device pointer
* @js: Job slot to use
*
* Caller must hold kbasep_jd_device_data::queue_mutex
*
* Return: Context to use for specified slot.
* NULL if no contexts present for specified slot
*/
static struct kbase_context *kbase_js_ctx_list_pop_head(
struct kbase_device *kbdev,
int js)
{
struct kbase_context *kctx;
lockdep_assert_held(&kbdev->js_data.queue_mutex);
if (list_empty(&kbdev->js_data.ctx_list_pullable[js]))
return NULL;
kctx = list_entry(kbdev->js_data.ctx_list_pullable[js].next,
struct kbase_context,
jctx.sched_info.ctx.ctx_list_entry[js]);
list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
return kctx;
}
/**
* kbase_js_ctx_pullable - Return if a context can be pulled from on the
* specified slot
* @kctx: Context pointer
* @js: Job slot to use
* @is_scheduled: true if the context is currently scheduled
*
* Caller must hold runpool_irq.lock
*
* Return: true if context can be pulled from on specified slot
* false otherwise
*/
static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js,
bool is_scheduled)
{
struct kbasep_js_device_data *js_devdata;
struct kbase_jd_atom *katom;
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
js_devdata = &kctx->kbdev->js_data;
if (is_scheduled) {
if (!kbasep_js_is_submit_allowed(js_devdata, kctx))
return false;
}
katom = jsctx_rb_peek(kctx, js);
if (!katom)
return false; /* No pullable atoms */
if (atomic_read(&katom->blocked))
return false; /* next atom blocked */
if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) {
if (katom->x_pre_dep->gpu_rb_state ==
KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB ||
katom->x_pre_dep->will_fail_event_code)
return false;
if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) &&
kbase_backend_nr_atoms_on_slot(kctx->kbdev, js))
return false;
}
return true;
}
static bool kbase_js_dep_validate(struct kbase_context *kctx,
struct kbase_jd_atom *katom)
{
struct kbase_device *kbdev = kctx->kbdev;
bool ret = true;
bool has_dep = false, has_x_dep = false;
int js = kbase_js_get_slot(kbdev, katom);
int prio = katom->sched_priority;
int i;
for (i = 0; i < 2; i++) {
struct kbase_jd_atom *dep_atom = katom->dep[i].atom;
if (dep_atom) {
int dep_js = kbase_js_get_slot(kbdev, dep_atom);
int dep_prio = dep_atom->sched_priority;
/* Dependent atom must already have been submitted */
if (!(dep_atom->atom_flags &
(KBASE_KATOM_FLAG_JSCTX_IN_LL |
KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED))){
ret = false;
break;
}
/* Dependencies with different priorities can't
be represented in the ringbuffer */
if (prio != dep_prio) {
ret = false;
break;
}
if (js == dep_js) {
/* Only one same-slot dependency can be
* represented in the ringbuffer */
if (has_dep) {
ret = false;
break;
}
has_dep = true;
} else {
/* Only one cross-slot dependency can be
* represented in the ringbuffer */
if (has_x_dep) {
ret = false;
break;
}
/* Each dependee atom can only have one
* cross-slot dependency */
if (dep_atom->x_post_dep) {
ret = false;
break;
}
/* The dependee atom can not already be in the
* HW access ringbuffer */
if (dep_atom->gpu_rb_state !=
KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
ret = false;
break;
}
/* The dependee atom can not already have
* completed */
if (dep_atom->status !=
KBASE_JD_ATOM_STATE_IN_JS) {
ret = false;
break;
}
/* Cross-slot dependencies must not violate
* PRLAM-8987 affinity restrictions */
if (kbase_hw_has_issue(kbdev,
BASE_HW_ISSUE_8987) &&
(js == 2 || dep_js == 2)) {
ret = false;
break;
}
has_x_dep = true;
}
if (kbase_jd_katom_dep_type(&katom->dep[i]) ==
BASE_JD_DEP_TYPE_DATA &&
js == dep_js) {
struct kbase_jd_atom *last_atom =
jsctx_rb_peek_last(kctx, js,
prio);
/* Last atom on slot must be pre-dep for this
* atom */
if (last_atom != dep_atom) {
ret = false;
break;
}
}
/* Dependency can be represented in ringbuffers */
}
}
/* If dependencies can be represented by ringbuffer then clear them from
* atom structure */
if (ret) {
for (i = 0; i < 2; i++) {
struct kbase_jd_atom *dep_atom = katom->dep[i].atom;
if (dep_atom) {
int dep_js = kbase_js_get_slot(kbdev, dep_atom);
if ((js != dep_js) &&
(dep_atom->status !=
KBASE_JD_ATOM_STATE_COMPLETED)
&& (dep_atom->status !=
KBASE_JD_ATOM_STATE_HW_COMPLETED)
&& (dep_atom->status !=
KBASE_JD_ATOM_STATE_UNUSED)) {
katom->atom_flags |=
KBASE_KATOM_FLAG_X_DEP_BLOCKED;
katom->x_pre_dep = dep_atom;
dep_atom->x_post_dep = katom;
if (kbase_jd_katom_dep_type(
&katom->dep[i]) ==
BASE_JD_DEP_TYPE_DATA)
katom->atom_flags |=
KBASE_KATOM_FLAG_FAIL_BLOCKER;
}
if ((kbase_jd_katom_dep_type(&katom->dep[i])
== BASE_JD_DEP_TYPE_DATA) &&
(js == dep_js))
katom->atom_flags |=
KBASE_KATOM_FLAG_FAIL_PREV;
list_del(&katom->dep_item[i]);
kbase_jd_katom_dep_clear(&katom->dep[i]);
}
}
}
return ret;
}
bool kbasep_js_add_job(struct kbase_context *kctx,
struct kbase_jd_atom *atom)
{
unsigned long flags;
struct kbasep_js_kctx_info *js_kctx_info;
struct kbase_device *kbdev;
struct kbasep_js_device_data *js_devdata;
union kbasep_js_policy *js_policy;
bool enqueue_required = false;
bool timer_sync = false;
KBASE_DEBUG_ASSERT(kctx != NULL);
KBASE_DEBUG_ASSERT(atom != NULL);
lockdep_assert_held(&kctx->jctx.lock);
kbdev = kctx->kbdev;
js_devdata = &kbdev->js_data;
js_policy = &kbdev->js_data.policy;
js_kctx_info = &kctx->jctx.sched_info;
mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
/*
* Begin Runpool transaction
*/
mutex_lock(&js_devdata->runpool_mutex);
/* Refcount ctx.nr_jobs */
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX);
++(js_kctx_info->ctx.nr_jobs);
/* Setup any scheduling information */
kbasep_js_clear_job_retry_submit(atom);
/* Lock for state available during IRQ */
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
if (!kbase_js_dep_validate(kctx, atom)) {
/* Dependencies could not be represented */
--(js_kctx_info->ctx.nr_jobs);
/* Setting atom status back to queued as it still has unresolved
* dependencies */
atom->status = KBASE_JD_ATOM_STATE_QUEUED;
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
mutex_unlock(&js_devdata->runpool_mutex);
goto out_unlock;
}
KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, atom));
enqueue_required = kbase_js_dep_resolved_submit(kctx, atom);
KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc,
kbasep_js_trace_get_refcnt_nolock(kbdev, kctx));
/* Context Attribute Refcounting */
kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom);
if (enqueue_required) {
if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false))
timer_sync = kbase_js_ctx_list_add_pullable(kbdev, kctx,
atom->slot_nr);
else
timer_sync = kbase_js_ctx_list_add_unpullable(kbdev,
kctx, atom->slot_nr);
}
/* If this context is active and the atom is the first on its slot,
* kick the job manager to attempt to fast-start the atom */
if (enqueue_required && kctx == kbdev->hwaccess.active_kctx)
kbase_jm_try_kick(kbdev, 1 << atom->slot_nr);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
if (timer_sync)
kbase_backend_ctx_count_changed(kbdev);
mutex_unlock(&js_devdata->runpool_mutex);
/* End runpool transaction */
if (!js_kctx_info->ctx.is_scheduled) {
if (js_kctx_info->ctx.is_dying) {
/* A job got added while/after kbase_job_zap_context()
* was called on a non-scheduled context (e.g. KDS
* dependency resolved). Kill that job by killing the
* context. */
kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx,
false);
} else if (js_kctx_info->ctx.nr_jobs == 1) {
/* Handle Refcount going from 0 to 1: schedule the
* context on the Policy Queue */
KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx);
/* Policy Queue was updated - caller must try to
* schedule the head context */
WARN_ON(!enqueue_required);
}
}
out_unlock:
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
return enqueue_required;
}
void kbasep_js_remove_job(struct kbase_device *kbdev,
struct kbase_context *kctx, struct kbase_jd_atom *atom)
{
struct kbasep_js_kctx_info *js_kctx_info;
struct kbasep_js_device_data *js_devdata;
union kbasep_js_policy *js_policy;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
KBASE_DEBUG_ASSERT(atom != NULL);
js_devdata = &kbdev->js_data;
js_policy = &kbdev->js_data.policy;
js_kctx_info = &kctx->jctx.sched_info;
KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc,
kbasep_js_trace_get_refcnt(kbdev, kctx));
/* De-refcount ctx.nr_jobs */
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0);
--(js_kctx_info->ctx.nr_jobs);
}
bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev,
struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
unsigned long flags;
struct kbasep_js_atom_retained_state katom_retained_state;
struct kbasep_js_device_data *js_devdata;
bool attr_state_changed;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
KBASE_DEBUG_ASSERT(katom != NULL);
js_devdata = &kbdev->js_data;
kbasep_js_atom_retained_state_copy(&katom_retained_state, katom);
kbasep_js_remove_job(kbdev, kctx, katom);
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
/* The atom has 'finished' (will not be re-run), so no need to call
* kbasep_js_has_atom_finished().
*
* This is because it returns false for soft-stopped atoms, but we
* want to override that, because we're cancelling an atom regardless of
* whether it was soft-stopped or not */
attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx,
&katom_retained_state);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
return attr_state_changed;
}
bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
bool result;
KBASE_DEBUG_ASSERT(kbdev != NULL);
js_devdata = &kbdev->js_data;
/* KBASE_TRACE_ADD_REFCOUNT( kbdev, JS_RETAIN_CTX, kctx, NULL, 0,
kbasep_js_trace_get_refcnt(kbdev, kctx)); */
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
return result;
}
struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev,
int as_nr)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
struct kbase_context *found_kctx = NULL;
struct kbasep_js_per_as_data *js_per_as_data;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS);
js_devdata = &kbdev->js_data;
js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
found_kctx = js_per_as_data->kctx;
if (found_kctx != NULL)
++(js_per_as_data->as_busy_refcount);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
return found_kctx;
}
struct kbase_context *kbasep_js_runpool_lookup_ctx_nolock(
struct kbase_device *kbdev, int as_nr)
{
struct kbasep_js_device_data *js_devdata;
struct kbase_context *found_kctx = NULL;
struct kbasep_js_per_as_data *js_per_as_data;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS);
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
js_devdata = &kbdev->js_data;
js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
found_kctx = js_per_as_data->kctx;
if (found_kctx != NULL)
++(js_per_as_data->as_busy_refcount);
return found_kctx;
}
/**
* kbasep_js_release_result - Try running more jobs after releasing a context
* and/or atom
*
* @kbdev: The kbase_device to operate on
* @kctx: The kbase_context to operate on
* @katom_retained_state: Retained state from the atom
* @runpool_ctx_attr_change: True if the runpool context attributes have changed
*
* This collates a set of actions that must happen whilst
* kbasep_js_device_data.runpool_irq.lock is held.
*
* This includes running more jobs when:
* - The previously released kctx caused a ctx attribute change,
* - The released atom caused a ctx attribute change,
* - Slots were previously blocked due to affinity restrictions,
* - Submission during IRQ handling failed.
*
* Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were
* changed. The caller should try scheduling all contexts
*/
static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release(
struct kbase_device *kbdev,
struct kbase_context *kctx,
struct kbasep_js_atom_retained_state *katom_retained_state,
bool runpool_ctx_attr_change)
{
struct kbasep_js_device_data *js_devdata;
kbasep_js_release_result result = 0;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
KBASE_DEBUG_ASSERT(katom_retained_state != NULL);
js_devdata = &kbdev->js_data;
lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
lockdep_assert_held(&js_devdata->runpool_mutex);
lockdep_assert_held(&js_devdata->runpool_irq.lock);
if (js_devdata->nr_user_contexts_running != 0) {
bool retry_submit = false;
int retry_jobslot = 0;
if (katom_retained_state)
retry_submit = kbasep_js_get_atom_retry_submit_slot(
katom_retained_state, &retry_jobslot);
if (runpool_ctx_attr_change || retry_submit) {
/* A change in runpool ctx attributes might mean we can
* run more jobs than before */
result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL;
KBASE_TRACE_ADD_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB,
kctx, NULL, 0u, retry_jobslot);
}
}
return result;
}
/*
* Internal function to release the reference on a ctx and an atom's "retained
* state", only taking the runpool and as transaction mutexes
*
* This also starts more jobs running in the case of an ctx-attribute state
* change
*
* This does none of the followup actions for scheduling:
* - It does not schedule in a new context
* - It does not requeue or handle dying contexts
*
* For those tasks, just call kbasep_js_runpool_release_ctx() instead
*
* Requires:
* - Context is scheduled in, and kctx->as_nr matches kctx_as_nr
* - Context has a non-zero refcount
* - Caller holds js_kctx_info->ctx.jsctx_mutex
* - Caller holds js_devdata->runpool_mutex
*/
static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
struct kbase_device *kbdev,
struct kbase_context *kctx,
struct kbasep_js_atom_retained_state *katom_retained_state)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
union kbasep_js_policy *js_policy;
struct kbasep_js_per_as_data *js_per_as_data;
kbasep_js_release_result release_result = 0u;
bool runpool_ctx_attr_change = false;
int kctx_as_nr;
struct kbase_as *current_as;
int new_ref_count;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
js_kctx_info = &kctx->jctx.sched_info;
js_devdata = &kbdev->js_data;
js_policy = &kbdev->js_data.policy;
/* Ensure context really is scheduled in */
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled);
/* kctx->as_nr and js_per_as_data are only read from here. The caller's
* js_ctx_mutex provides a barrier that ensures they are up-to-date.
*
* They will not change whilst we're reading them, because the refcount
* is non-zero (and we ASSERT on that last fact).
*/
kctx_as_nr = kctx->as_nr;
KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID);
js_per_as_data = &js_devdata->runpool_irq.per_as_data[kctx_as_nr];
KBASE_DEBUG_ASSERT(js_per_as_data->as_busy_refcount > 0);
/*
* Transaction begins on AS and runpool_irq
*
* Assert about out calling contract
*/
current_as = &kbdev->as[kctx_as_nr];
mutex_lock(&kbdev->pm.lock);
mutex_lock(&current_as->transaction_mutex);
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr);
KBASE_DEBUG_ASSERT(js_per_as_data->as_busy_refcount > 0);
/* Update refcount */
new_ref_count = --(js_per_as_data->as_busy_refcount);
/* Release the atom if it finished (i.e. wasn't soft-stopped) */
if (kbasep_js_has_atom_finished(katom_retained_state))
runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom(
kbdev, kctx, katom_retained_state);
KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u,
new_ref_count);
if (new_ref_count == 1 && kctx->jctx.sched_info.ctx.flags &
KBASE_CTX_FLAG_PRIVILEGED &&
!kbase_pm_is_suspending(kbdev)) {
/* Context is kept scheduled into an address space even when
* there are no jobs, in this case we have to handle the
* situation where all jobs have been evicted from the GPU and
* submission is disabled.
*
* At this point we re-enable submission to allow further jobs
* to be executed
*/
kbasep_js_set_submit_allowed(js_devdata, kctx);
}
/* Make a set of checks to see if the context should be scheduled out */
if (new_ref_count == 0 &&
(!kbasep_js_is_submit_allowed(js_devdata, kctx) ||
kbdev->pm.suspending)) {
/* Last reference, and we've been told to remove this context
* from the Run Pool */
dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because as_busy_refcount=%d, jobs=%d, allowed=%d",
kctx, new_ref_count, js_kctx_info->ctx.nr_jobs,
kbasep_js_is_submit_allowed(js_devdata, kctx));
#if defined(CONFIG_MALI_GATOR_SUPPORT)
kbase_trace_mali_mmu_as_released(kctx->as_nr);
#endif
kbase_tlstream_tl_nret_as_ctx(&kbdev->as[kctx->as_nr], kctx);
kbase_backend_release_ctx_irq(kbdev, kctx);
if (kbdev->hwaccess.active_kctx == kctx)
kbdev->hwaccess.active_kctx = NULL;
/* Ctx Attribute handling
*
* Releasing atoms attributes must either happen before this, or
* after 'is_scheduled' is changed, otherwise we double-decount
* the attributes */
runpool_ctx_attr_change |=
kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx);
/* Releasing the context and katom retained state can allow
* more jobs to run */
release_result |=
kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev,
kctx, katom_retained_state,
runpool_ctx_attr_change);
/*
* Transaction ends on AS and runpool_irq:
*
* By this point, the AS-related data is now clear and ready
* for re-use.
*
* Since releases only occur once for each previous successful
* retain, and no more retains are allowed on this context, no
* other thread will be operating in this
* code whilst we are
*/
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
kbase_backend_release_ctx_noirq(kbdev, kctx);
mutex_unlock(&current_as->transaction_mutex);
mutex_unlock(&kbdev->pm.lock);
/* Note: Don't reuse kctx_as_nr now */
/* Synchronize with any policy timers */
kbase_backend_ctx_count_changed(kbdev);
/* update book-keeping info */
js_kctx_info->ctx.is_scheduled = false;
/* Signal any waiter that the context is not scheduled, so is
* safe for termination - once the jsctx_mutex is also dropped,
* and jobs have finished. */
wake_up(&js_kctx_info->ctx.is_scheduled_wait);
/* Queue an action to occur after we've dropped the lock */
release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED;
} else {
kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx,
katom_retained_state, runpool_ctx_attr_change);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
mutex_unlock(&current_as->transaction_mutex);
mutex_unlock(&kbdev->pm.lock);
}
return release_result;
}
void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_atom_retained_state katom_retained_state;
/* Setup a dummy katom_retained_state */
kbasep_js_atom_retained_state_init_invalid(&katom_retained_state);
kbasep_js_runpool_release_ctx_internal(kbdev, kctx,
&katom_retained_state);
}
void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx, bool has_pm_ref)
{
struct kbasep_js_device_data *js_devdata;
union kbasep_js_policy *js_policy;
struct kbasep_js_kctx_info *js_kctx_info;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
js_kctx_info = &kctx->jctx.sched_info;
js_policy = &kbdev->js_data.policy;
js_devdata = &kbdev->js_data;
/* This is called if and only if you've you've detached the context from
* the Runpool or the Policy Queue, and not added it back to the Runpool
*/
KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
if (js_kctx_info->ctx.is_dying) {
/* Dying: don't requeue, but kill all jobs on the context. This
* happens asynchronously */
dev_dbg(kbdev->dev,
"JS: ** Killing Context %p on RunPool Remove **", kctx);
kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel);
}
}
void kbasep_js_runpool_release_ctx_and_katom_retained_state(
struct kbase_device *kbdev, struct kbase_context *kctx,
struct kbasep_js_atom_retained_state *katom_retained_state)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
base_jd_event_code event_code;
kbasep_js_release_result release_result;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
js_kctx_info = &kctx->jctx.sched_info;
js_devdata = &kbdev->js_data;
event_code = katom_retained_state->event_code;
mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx,
katom_retained_state);
/* Drop the runpool mutex to allow requeing kctx */
mutex_unlock(&js_devdata->runpool_mutex);
if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u)
kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true);
/* Drop the jsctx_mutex to allow scheduling in a new context */
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL)
kbase_js_sched_all(kbdev);
}
void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_atom_retained_state katom_retained_state;
kbasep_js_atom_retained_state_init_invalid(&katom_retained_state);
kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx,
&katom_retained_state);
}
/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into
* kbase_js_sched_all() */
static void kbasep_js_runpool_release_ctx_no_schedule(
struct kbase_device *kbdev, struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
kbasep_js_release_result release_result;
struct kbasep_js_atom_retained_state katom_retained_state_struct;
struct kbasep_js_atom_retained_state *katom_retained_state =
&katom_retained_state_struct;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
js_kctx_info = &kctx->jctx.sched_info;
js_devdata = &kbdev->js_data;
kbasep_js_atom_retained_state_init_invalid(katom_retained_state);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx,
katom_retained_state);
/* Drop the runpool mutex to allow requeing kctx */
mutex_unlock(&js_devdata->runpool_mutex);
if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u)
kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true);
/* Drop the jsctx_mutex to allow scheduling in a new context */
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
/* NOTE: could return release_result if the caller would like to know
* whether it should schedule a new context, but currently no callers do
*/
}
/**
* kbase_js_set_timeouts - update all JS timeouts with user specified data
* @kbdev: Device pointer
*
* Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is
* set to a positive number then that becomes the new value used, if a timeout
* is negative then the default is set.
*/
static void kbase_js_set_timeouts(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_data = &kbdev->js_data;
if (kbdev->js_scheduling_period_ns < 0)
js_data->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS;
else if (kbdev->js_scheduling_period_ns > 0)
js_data->scheduling_period_ns = kbdev->js_scheduling_period_ns;
if (kbdev->js_soft_stop_ticks < 0)
js_data->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS;
else if (kbdev->js_soft_stop_ticks > 0)
js_data->soft_stop_ticks = kbdev->js_soft_stop_ticks;
if (kbdev->js_soft_stop_ticks_cl < 0)
js_data->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL;
else if (kbdev->js_soft_stop_ticks_cl > 0)
js_data->soft_stop_ticks_cl = kbdev->js_soft_stop_ticks_cl;
if (kbdev->js_hard_stop_ticks_ss < 0) {
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
js_data->hard_stop_ticks_ss =
DEFAULT_JS_HARD_STOP_TICKS_SS_8408;
else
js_data->hard_stop_ticks_ss =
DEFAULT_JS_HARD_STOP_TICKS_SS;
} else if (kbdev->js_hard_stop_ticks_ss > 0) {
js_data->hard_stop_ticks_ss = kbdev->js_hard_stop_ticks_ss;
}
if (kbdev->js_hard_stop_ticks_cl < 0)
js_data->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL;
else if (kbdev->js_hard_stop_ticks_cl > 0)
js_data->hard_stop_ticks_cl = kbdev->js_hard_stop_ticks_cl;
if (kbdev->js_hard_stop_ticks_dumping < 0)
js_data->hard_stop_ticks_dumping =
DEFAULT_JS_HARD_STOP_TICKS_DUMPING;
else if (kbdev->js_hard_stop_ticks_dumping > 0)
js_data->hard_stop_ticks_dumping =
kbdev->js_hard_stop_ticks_dumping;
if (kbdev->js_reset_ticks_ss < 0) {
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
js_data->gpu_reset_ticks_ss =
DEFAULT_JS_RESET_TICKS_SS_8408;
else
js_data->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS;
} else if (kbdev->js_reset_ticks_ss > 0) {
js_data->gpu_reset_ticks_ss = kbdev->js_reset_ticks_ss;
}
if (kbdev->js_reset_ticks_cl < 0)
js_data->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL;
else if (kbdev->js_reset_ticks_cl > 0)
js_data->gpu_reset_ticks_cl = kbdev->js_reset_ticks_cl;
if (kbdev->js_reset_ticks_dumping < 0)
js_data->gpu_reset_ticks_dumping =
DEFAULT_JS_RESET_TICKS_DUMPING;
else if (kbdev->js_reset_ticks_dumping > 0)
js_data->gpu_reset_ticks_dumping =
kbdev->js_reset_ticks_dumping;
}
static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
union kbasep_js_policy *js_policy;
struct kbase_as *new_address_space = NULL;
unsigned long flags;
bool kctx_suspended = false;
int as_nr;
js_devdata = &kbdev->js_data;
js_policy = &kbdev->js_data.policy;
js_kctx_info = &kctx->jctx.sched_info;
/* Pick available address space for this context */
as_nr = kbase_backend_find_free_address_space(kbdev, kctx);
if (as_nr == KBASEP_AS_NR_INVALID)
return false; /* No address spaces currently available */
new_address_space = &kbdev->as[as_nr];
/*
* Atomic transaction on the Context and Run Pool begins
*/
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
/* Check to see if context is dying due to kbase_job_zap_context() */
if (js_kctx_info->ctx.is_dying) {
/* Roll back the transaction so far and return */
kbase_backend_release_free_address_space(kbdev, as_nr);
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
return false;
}
KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL,
0u,
kbasep_js_trace_get_refcnt(kbdev, kctx));
if (js_devdata->nr_user_contexts_running == 0 &&
kbdev->js_timeouts_updated) {
/* Only when there are no other contexts submitting jobs:
* Latch in run-time job scheduler timeouts that were set
* through js_timeouts sysfs file */
kbase_js_set_timeouts(kbdev);
kbdev->js_timeouts_updated = false;
}
js_kctx_info->ctx.is_scheduled = true;
mutex_lock(&new_address_space->transaction_mutex);
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
/* Assign context to previously chosen address space */
if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) {
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
mutex_unlock(&new_address_space->transaction_mutex);
/* If address space is not pending, then kbase_backend_use_ctx()
* failed. Roll back the transaction so far and return */
if (!kctx->as_pending) {
js_kctx_info->ctx.is_scheduled = false;
kbase_backend_release_free_address_space(kbdev, as_nr);
}
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
return false;
}
kbdev->hwaccess.active_kctx = kctx;
#if defined(CONFIG_MALI_GATOR_SUPPORT)
kbase_trace_mali_mmu_as_in_use(kctx->as_nr);
#endif
kbase_tlstream_tl_ret_as_ctx(&kbdev->as[kctx->as_nr], kctx);
/* Cause any future waiter-on-termination to wait until the context is
* descheduled */
wake_up(&js_kctx_info->ctx.is_scheduled_wait);
/* Re-check for suspending: a suspend could've occurred, and all the
* contexts could've been removed from the runpool before we took this
* lock. In this case, we don't want to allow this context to run jobs,
* we just want it out immediately.
*
* The DMB required to read the suspend flag was issued recently as part
* of the runpool_irq locking. If a suspend occurs *after* that lock was
* taken (i.e. this condition doesn't execute), then the
* kbasep_js_suspend() code will cleanup this context instead (by virtue
* of it being called strictly after the suspend flag is set, and will
* wait for this lock to drop) */
if (kbase_pm_is_suspending(kbdev)) {
/* Cause it to leave at some later point */
bool retained;
retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
KBASE_DEBUG_ASSERT(retained);
kbasep_js_clear_submit_allowed(js_devdata, kctx);
kctx_suspended = true;
}
/* Transaction complete */
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
mutex_unlock(&new_address_space->transaction_mutex);
/* Synchronize with any policy timers */
kbase_backend_ctx_count_changed(kbdev);
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
/* Note: after this point, the context could potentially get scheduled
* out immediately */
if (kctx_suspended) {
/* Finishing forcing out the context due to a suspend. Use a
* variant of kbasep_js_runpool_release_ctx() that doesn't
* schedule a new context, to prevent a risk of recursion back
* into this function */
kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx);
return false;
}
return true;
}
static bool kbase_js_use_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
unsigned long flags;
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
if (kctx->as_pending) {
/* Context waiting for AS to be assigned */
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
return false;
}
if (kbase_backend_use_ctx_sched(kbdev, kctx)) {
/* Context already has ASID - mark as active */
kbdev->hwaccess.active_kctx = kctx;
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
return true; /* Context already scheduled */
}
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
return kbasep_js_schedule_ctx(kbdev, kctx);
}
void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_kctx_info *js_kctx_info;
struct kbasep_js_device_data *js_devdata;
bool is_scheduled;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
js_devdata = &kbdev->js_data;
js_kctx_info = &kctx->jctx.sched_info;
/* This must never be attempted whilst suspending - i.e. it should only
* happen in response to a syscall from a user-space thread */
BUG_ON(kbase_pm_is_suspending(kbdev));
mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
/* Mark the context as privileged */
js_kctx_info->ctx.flags |= KBASE_CTX_FLAG_PRIVILEGED;
is_scheduled = js_kctx_info->ctx.is_scheduled;
if (!is_scheduled) {
/* Add the context to the pullable list */
if (kbase_js_ctx_list_add_pullable(kbdev, kctx, 0))
kbase_js_sync_timers(kbdev);
/* Fast-starting requires the jsctx_mutex to be dropped,
* because it works on multiple ctxs */
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
/* Try to schedule the context in */
kbase_js_sched_all(kbdev);
/* Wait for the context to be scheduled in */
wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait,
kctx->jctx.sched_info.ctx.is_scheduled);
} else {
/* Already scheduled in - We need to retain it to keep the
* corresponding address space */
kbasep_js_runpool_retain_ctx(kbdev, kctx);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
}
}
KBASE_EXPORT_TEST_API(kbasep_js_schedule_privileged_ctx);
void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_kctx_info *js_kctx_info;
bool pending;
KBASE_DEBUG_ASSERT(kctx != NULL);
js_kctx_info = &kctx->jctx.sched_info;
/* We don't need to use the address space anymore */
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
js_kctx_info->ctx.flags &= (~KBASE_CTX_FLAG_PRIVILEGED);
pending = kctx->as_pending;
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
/* Release the context - it will be scheduled out if there is no
* pending job */
if (!pending)
kbasep_js_runpool_release_ctx(kbdev, kctx);
kbase_js_sched_all(kbdev);
}
KBASE_EXPORT_TEST_API(kbasep_js_release_privileged_ctx);
void kbasep_js_suspend(struct kbase_device *kbdev)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
int i;
u16 retained = 0u;
int nr_privileged_ctx = 0;
KBASE_DEBUG_ASSERT(kbdev);
KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev));
js_devdata = &kbdev->js_data;
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
/* Prevent all contexts from submitting */
js_devdata->runpool_irq.submit_allowed = 0;
/* Retain each of the contexts, so we can cause it to leave even if it
* had no refcount to begin with */
for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) {
struct kbasep_js_per_as_data *js_per_as_data =
&js_devdata->runpool_irq.per_as_data[i];
struct kbase_context *kctx = js_per_as_data->kctx;
retained = retained << 1;
if (kctx) {
++(js_per_as_data->as_busy_refcount);
retained |= 1u;
/* We can only cope with up to 1 privileged context -
* the instrumented context. It'll be suspended by
* disabling instrumentation */
if (kctx->jctx.sched_info.ctx.flags &
KBASE_CTX_FLAG_PRIVILEGED)
KBASE_DEBUG_ASSERT(++nr_privileged_ctx == 1);
}
}
CSTD_UNUSED(nr_privileged_ctx);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
/* De-ref the previous retain to ensure each context gets pulled out
* sometime later. */
for (i = 0;
i < BASE_MAX_NR_AS;
++i, retained = retained >> 1) {
struct kbasep_js_per_as_data *js_per_as_data =
&js_devdata->runpool_irq.per_as_data[i];
struct kbase_context *kctx = js_per_as_data->kctx;
if (retained & 1u)
kbasep_js_runpool_release_ctx(kbdev, kctx);
}
/* Caller must wait for all Power Manager active references to be
* dropped */
}
void kbasep_js_resume(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_devdata;
int js;
KBASE_DEBUG_ASSERT(kbdev);
js_devdata = &kbdev->js_data;
KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
mutex_lock(&js_devdata->queue_mutex);
for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
struct kbase_context *kctx, *n;
list_for_each_entry_safe(kctx, n,
&kbdev->js_data.ctx_list_unpullable[js],
jctx.sched_info.ctx.ctx_list_entry[js]) {
struct kbasep_js_kctx_info *js_kctx_info;
unsigned long flags;
bool timer_sync = false;
js_kctx_info = &kctx->jctx.sched_info;
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
if (!js_kctx_info->ctx.is_scheduled &&
kbase_js_ctx_pullable(kctx, js, false))
timer_sync = kbase_js_ctx_list_add_pullable(
kbdev, kctx, js);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
flags);
if (timer_sync)
kbase_backend_ctx_count_changed(kbdev);
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
}
}
mutex_unlock(&js_devdata->queue_mutex);
/* Restart atom processing */
kbase_js_sched_all(kbdev);
/* JS Resume complete */
}
bool kbase_js_is_atom_valid(struct kbase_device *kbdev,
struct kbase_jd_atom *katom)
{
if ((katom->core_req & BASE_JD_REQ_FS) &&
(katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE |
BASE_JD_REQ_T)))
return false;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987) &&
(katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) &&
(katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_T)))
return false;
return true;
}
static int kbase_js_get_slot(struct kbase_device *kbdev,
struct kbase_jd_atom *katom)
{
if (katom->core_req & BASE_JD_REQ_FS)
return 0;
if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
if (katom->device_nr == 1 &&
kbdev->gpu_props.num_core_groups == 2)
return 2;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
return 2;
}
return 1;
}
bool kbase_js_dep_resolved_submit(struct kbase_context *kctx,
struct kbase_jd_atom *katom)
{
bool enqueue_required;
katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom);
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
lockdep_assert_held(&kctx->jctx.lock);
/* If slot will transition from unpullable to pullable then add to
* pullable list */
if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) {
enqueue_required = true;
} else {
enqueue_required = false;
}
/* Check if there are lower priority jobs to soft stop */
kbase_job_slot_ctx_priority_check_locked(kctx, katom);
/* Add atom to ring buffer. */
if (jsctx_rb_add_atom(kctx, katom)) {
jsctx_ll_add(kctx, katom);
enqueue_required = false;
katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_LL;
} else {
katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED;
}
return enqueue_required;
}
/**
* kbase_js_evict_deps - Evict dependencies of a failed atom.
* @kctx: Context pointer
* @katom: Pointer to the atom that has failed.
* @js: The job slot the katom was run on.
* @prio: Priority of the katom.
*
* Remove all post dependencies of an atom from the context ringbuffers.
*
* The original atom's event_code will be propogated to all dependent atoms.
*
* Context: Caller must hold the HW access lock
*/
static void kbase_js_evict_deps(struct kbase_context *kctx,
struct kbase_jd_atom *katom, int js, int prio)
{
struct kbase_jd_atom *x_dep = katom->x_post_dep;
struct kbase_jd_atom *next_katom = jsctx_rb_peek_prio(kctx, js, prio);
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
if (next_katom &&
(next_katom->atom_flags & KBASE_KATOM_FLAG_FAIL_PREV)) {
KBASE_DEBUG_ASSERT(next_katom->status !=
KBASE_JD_ATOM_STATE_HW_COMPLETED);
next_katom->will_fail_event_code = katom->event_code;
}
/* Has cross slot depenency. */
if (x_dep && (x_dep->atom_flags & (KBASE_KATOM_FLAG_JSCTX_IN_LL |
KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED))) {
/* Remove dependency.*/
x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED;
/* Fail if it had a data dependency. */
if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) {
x_dep->will_fail_event_code = katom->event_code;
}
}
}
struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js)
{
struct kbase_jd_atom *katom;
struct kbasep_js_device_data *js_devdata;
int pulled;
KBASE_DEBUG_ASSERT(kctx);
js_devdata = &kctx->kbdev->js_data;
lockdep_assert_held(&js_devdata->runpool_irq.lock);
if (!kbasep_js_is_submit_allowed(js_devdata, kctx))
return NULL;
if (kbase_pm_is_suspending(kctx->kbdev))
return NULL;
katom = jsctx_rb_peek(kctx, js);
if (!katom)
return NULL;
if (atomic_read(&katom->blocked))
return NULL;
/* Due to ordering restrictions when unpulling atoms on failure, we do
* not allow multiple runs of fail-dep atoms from the same context to be
* present on the same slot */
if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_PREV) &&
atomic_read(&kctx->atoms_pulled_slot[js])) {
struct kbase_jd_atom *prev_atom =
kbase_backend_inspect_tail(kctx->kbdev, js);
if (prev_atom && prev_atom->kctx != kctx)
return NULL;
}
if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) {
if (katom->x_pre_dep->gpu_rb_state ==
KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB ||
katom->x_pre_dep->will_fail_event_code)
return NULL;
if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) &&
kbase_backend_nr_atoms_on_slot(kctx->kbdev, js))
return NULL;
}
kctx->pulled = true;
pulled = atomic_inc_return(&kctx->atoms_pulled);
if (pulled == 1 && !kctx->slots_pullable) {
WARN_ON(kctx->ctx_runnable_ref);
kctx->ctx_runnable_ref = true;
atomic_inc(&kctx->kbdev->js_data.nr_contexts_runnable);
}
atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]);
jsctx_rb_pull(kctx, katom);
kbasep_js_runpool_retain_ctx_nolock(kctx->kbdev, kctx);
katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF;
katom->sched_info.cfs.ticks = 0;
return katom;
}
static void js_return_worker(struct work_struct *data)
{
struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom,
work);
struct kbase_context *kctx = katom->kctx;
struct kbase_device *kbdev = kctx->kbdev;
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info;
struct kbasep_js_atom_retained_state retained_state;
int js = katom->slot_nr;
bool timer_sync = false;
bool context_idle = false;
unsigned long flags;
base_jd_core_req core_req = katom->core_req;
u64 affinity = katom->affinity;
enum kbase_atom_coreref_state coreref_state = katom->coreref_state;
kbase_tlstream_aux_job_softstop_ex(katom);
kbase_backend_complete_wq(kbdev, katom);
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316))
kbase_as_poking_timer_release_atom(kbdev, kctx, katom);
kbasep_js_atom_retained_state_copy(&retained_state, katom);
mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
atomic_dec(&kctx->atoms_pulled);
atomic_dec(&kctx->atoms_pulled_slot[js]);
atomic_dec(&katom->blocked);
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
if (!atomic_read(&kctx->atoms_pulled_slot[js]) &&
jsctx_rb_none_to_pull(kctx, js))
timer_sync |= kbase_js_ctx_list_remove(kbdev, kctx, js);
if (!atomic_read(&kctx->atoms_pulled)) {
if (!kctx->slots_pullable) {
WARN_ON(!kctx->ctx_runnable_ref);
kctx->ctx_runnable_ref = false;
atomic_dec(&kbdev->js_data.nr_contexts_runnable);
timer_sync = true;
}
if (kctx->as_nr != KBASEP_AS_NR_INVALID &&
!js_kctx_info->ctx.is_dying) {
int num_slots = kbdev->gpu_props.num_job_slots;
int slot;
if (!kbasep_js_is_submit_allowed(js_devdata, kctx))
kbasep_js_set_submit_allowed(js_devdata, kctx);
for (slot = 0; slot < num_slots; slot++) {
if (kbase_js_ctx_pullable(kctx, slot, true))
timer_sync |=
kbase_js_ctx_list_add_pullable(
kbdev, kctx, slot);
}
}
kbase_jm_idle_ctx(kbdev, kctx);
context_idle = true;
}
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
if (context_idle) {
WARN_ON(!kctx->ctx_active);
kctx->ctx_active = false;
kbase_pm_context_idle(kbdev);
}
if (timer_sync)
kbase_js_sync_timers(kbdev);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF;
kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx,
&retained_state);
kbase_js_sched_all(kbdev);
kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity,
coreref_state);
}
void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
jsctx_rb_unpull(kctx, katom);
WARN_ON(work_pending(&katom->work));
/* Block re-submission until workqueue has run */
atomic_inc(&katom->blocked);
kbase_job_check_leave_disjoint(kctx->kbdev, katom);
KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
INIT_WORK(&katom->work, js_return_worker);
queue_work(kctx->jctx.job_done_wq, &katom->work);
}
bool kbase_js_complete_atom_wq(struct kbase_context *kctx,
struct kbase_jd_atom *katom)
{
struct kbasep_js_kctx_info *js_kctx_info;
struct kbasep_js_device_data *js_devdata;
struct kbase_device *kbdev;
unsigned long flags;
bool timer_sync = false;
int atom_slot;
bool context_idle = false;
kbdev = kctx->kbdev;
atom_slot = katom->slot_nr;
js_kctx_info = &kctx->jctx.sched_info;
js_devdata = &kbdev->js_data;
lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED) {
jsctx_rb_remove(kctx, katom);
context_idle = !atomic_dec_return(&kctx->atoms_pulled);
atomic_dec(&kctx->atoms_pulled_slot[atom_slot]);
if (!atomic_read(&kctx->atoms_pulled) &&
!kctx->slots_pullable) {
WARN_ON(!kctx->ctx_runnable_ref);
kctx->ctx_runnable_ref = false;
atomic_dec(&kbdev->js_data.nr_contexts_runnable);
timer_sync = true;
}
}
WARN_ON(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_LL);
if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) &&
jsctx_rb_none_to_pull(kctx, atom_slot))
timer_sync |= kbase_js_ctx_list_remove(kctx->kbdev, kctx,
atom_slot);
/*
* If submission is disabled on this context (most likely due to an
* atom failure) and there are now no atoms left in the system then
* re-enable submission so that context can be scheduled again.
*/
if (!kbasep_js_is_submit_allowed(js_devdata, kctx) &&
!atomic_read(&kctx->atoms_pulled) &&
!js_kctx_info->ctx.is_dying) {
int js;
kbasep_js_set_submit_allowed(js_devdata, kctx);
for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
if (kbase_js_ctx_pullable(kctx, js, true))
timer_sync |= kbase_js_ctx_list_add_pullable(
kbdev, kctx, js);
}
} else if (katom->x_post_dep &&
kbasep_js_is_submit_allowed(js_devdata, kctx)) {
int js;
for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
if (kbase_js_ctx_pullable(kctx, js, true))
timer_sync |= kbase_js_ctx_list_add_pullable(
kbdev, kctx, js);
}
}
/* Mark context as inactive. The pm reference will be dropped later in
* jd_done_worker().
*/
if (context_idle)
kctx->ctx_active = false;
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
if (timer_sync)
kbase_backend_ctx_count_changed(kbdev);
mutex_unlock(&js_devdata->runpool_mutex);
return context_idle;
}
void kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp)
{
u64 microseconds_spent = 0;
struct kbase_device *kbdev;
struct kbase_context *kctx = katom->kctx;
union kbasep_js_policy *js_policy;
struct kbasep_js_device_data *js_devdata;
kbdev = kctx->kbdev;
js_policy = &kbdev->js_data.policy;
js_devdata = &kbdev->js_data;
lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
if (katom->will_fail_event_code)
katom->event_code = katom->will_fail_event_code;
katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED;
if (katom->event_code != BASE_JD_EVENT_DONE) {
kbase_js_evict_deps(kctx, katom, katom->slot_nr,
katom->sched_priority);
}
#if defined(CONFIG_MALI_GATOR_SUPPORT)
kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_STOP,
katom->slot_nr), NULL, 0);
#endif
kbase_tlstream_tl_nret_atom_lpu(
katom,
&kbdev->gpu_props.props.raw_props.js_features[
katom->slot_nr]);
kbase_tlstream_tl_nret_atom_as(katom, &kbdev->as[kctx->as_nr]);
kbase_tlstream_tl_nret_ctx_lpu(
kctx,
&kbdev->gpu_props.props.raw_props.js_features[
katom->slot_nr]);
/* Calculate the job's time used */
if (end_timestamp != NULL) {
/* Only calculating it for jobs that really run on the HW (e.g.
* removed from next jobs never actually ran, so really did take
* zero time) */
ktime_t tick_diff = ktime_sub(*end_timestamp,
katom->start_timestamp);
microseconds_spent = ktime_to_ns(tick_diff);
do_div(microseconds_spent, 1000);
/* Round up time spent to the minimum timer resolution */
if (microseconds_spent < KBASEP_JS_TICK_RESOLUTION_US)
microseconds_spent = KBASEP_JS_TICK_RESOLUTION_US;
}
/* Log the result of the job (completion status, and time spent). */
kbasep_js_policy_log_job_result(js_policy, katom, microseconds_spent);
kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0);
/* Unblock cross dependency if present */
if (katom->x_post_dep && (katom->event_code == BASE_JD_EVENT_DONE ||
!(katom->x_post_dep->atom_flags &
KBASE_KATOM_FLAG_FAIL_BLOCKER)))
katom->x_post_dep->atom_flags &=
~KBASE_KATOM_FLAG_X_DEP_BLOCKED;
}
void kbase_js_sched(struct kbase_device *kbdev, int js_mask)
{
struct kbasep_js_device_data *js_devdata;
union kbasep_js_policy *js_policy;
bool timer_sync = false;
js_devdata = &kbdev->js_data;
js_policy = &js_devdata->policy;
down(&js_devdata->schedule_sem);
mutex_lock(&js_devdata->queue_mutex);
while (js_mask) {
int js;
js = ffs(js_mask) - 1;
while (1) {
struct kbase_context *kctx;
unsigned long flags;
bool context_idle = false;
kctx = kbase_js_ctx_list_pop_head(kbdev, js);
if (!kctx) {
js_mask &= ~(1 << js);
break; /* No contexts on pullable list */
}
if (!kctx->ctx_active) {
context_idle = true;
if (kbase_pm_context_active_handle_suspend(
kbdev,
KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) {
/* Suspend pending - return context to
* queue and stop scheduling */
mutex_lock(
&kctx->jctx.sched_info.ctx.jsctx_mutex);
if (kbase_js_ctx_list_add_pullable_head(
kctx->kbdev, kctx, js))
kbase_js_sync_timers(kbdev);
mutex_unlock(
&kctx->jctx.sched_info.ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
up(&js_devdata->schedule_sem);
return;
}
kctx->ctx_active = true;
}
if (!kbase_js_use_ctx(kbdev, kctx)) {
mutex_lock(
&kctx->jctx.sched_info.ctx.jsctx_mutex);
/* Context can not be used at this time */
spin_lock_irqsave(&js_devdata->runpool_irq.lock,
flags);
if (kctx->as_pending ||
kbase_js_ctx_pullable(kctx, js, false)
|| (kctx->jctx.sched_info.ctx.flags &
KBASE_CTX_FLAG_PRIVILEGED))
timer_sync |=
kbase_js_ctx_list_add_pullable_head(
kctx->kbdev, kctx, js);
else
timer_sync |=
kbase_js_ctx_list_add_unpullable(
kctx->kbdev, kctx, js);
spin_unlock_irqrestore(
&js_devdata->runpool_irq.lock, flags);
mutex_unlock(
&kctx->jctx.sched_info.ctx.jsctx_mutex);
if (context_idle) {
WARN_ON(!kctx->ctx_active);
kctx->ctx_active = false;
kbase_pm_context_idle(kbdev);
}
/* No more jobs can be submitted on this slot */
js_mask &= ~(1 << js);
break;
}
mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
kctx->pulled = false;
if (!kbase_jm_kick(kbdev, 1 << js))
/* No more jobs can be submitted on this slot */
js_mask &= ~(1 << js);
if (!kctx->pulled) {
/* Failed to pull jobs - push to head of list */
if (kbase_js_ctx_pullable(kctx, js, true))
timer_sync |=
kbase_js_ctx_list_add_pullable_head(
kctx->kbdev,
kctx, js);
else
timer_sync |=
kbase_js_ctx_list_add_unpullable(
kctx->kbdev,
kctx, js);
if (context_idle) {
kbase_jm_idle_ctx(kbdev, kctx);
spin_unlock_irqrestore(
&js_devdata->runpool_irq.lock,
flags);
WARN_ON(!kctx->ctx_active);
kctx->ctx_active = false;
kbase_pm_context_idle(kbdev);
} else {
spin_unlock_irqrestore(
&js_devdata->runpool_irq.lock,
flags);
}
mutex_unlock(
&kctx->jctx.sched_info.ctx.jsctx_mutex);
js_mask &= ~(1 << js);
break; /* Could not run atoms on this slot */
}
/* Push to back of list */
if (kbase_js_ctx_pullable(kctx, js, true))
timer_sync |= kbase_js_ctx_list_add_pullable(
kctx->kbdev, kctx, js);
else
timer_sync |= kbase_js_ctx_list_add_unpullable(
kctx->kbdev, kctx, js);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
flags);
mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
}
}
if (timer_sync)
kbase_js_sync_timers(kbdev);
mutex_unlock(&js_devdata->queue_mutex);
up(&js_devdata->schedule_sem);
}
void kbase_js_zap_context(struct kbase_context *kctx)
{
struct kbase_device *kbdev = kctx->kbdev;
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info;
int js;
/*
* Critical assumption: No more submission is possible outside of the
* workqueue. This is because the OS *must* prevent U/K calls (IOCTLs)
* whilst the struct kbase_context is terminating.
*/
/* First, atomically do the following:
* - mark the context as dying
* - try to evict it from the policy queue */
mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
js_kctx_info->ctx.is_dying = true;
dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx);
/*
* At this point we know:
* - If eviction succeeded, it was in the policy queue, but now no
* longer is
* - We must cancel the jobs here. No Power Manager active reference to
* release.
* - This happens asynchronously - kbase_jd_zap_context() will wait for
* those jobs to be killed.
* - If eviction failed, then it wasn't in the policy queue. It is one
* of the following:
* - a. it didn't have any jobs, and so is not in the Policy Queue or
* the Run Pool (not scheduled)
* - Hence, no more work required to cancel jobs. No Power Manager
* active reference to release.
* - b. it was in the middle of a scheduling transaction (and thus must
* have at least 1 job). This can happen from a syscall or a
* kernel thread. We still hold the jsctx_mutex, and so the thread
* must be waiting inside kbasep_js_try_schedule_head_ctx(),
* before checking whether the runpool is full. That thread will
* continue after we drop the mutex, and will notice the context
* is dying. It will rollback the transaction, killing all jobs at
* the same time. kbase_jd_zap_context() will wait for those jobs
* to be killed.
* - Hence, no more work required to cancel jobs, or to release the
* Power Manager active reference.
* - c. it is scheduled, and may or may not be running jobs
* - We must cause it to leave the runpool by stopping it from
* submitting any more jobs. When it finally does leave,
* kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs
* (because it is dying), release the Power Manager active reference,
* and will not requeue the context in the policy queue.
* kbase_jd_zap_context() will wait for those jobs to be killed.
* - Hence, work required just to make it leave the runpool. Cancelling
* jobs and releasing the Power manager active reference will be
* handled when it leaves the runpool.
*/
if (!js_kctx_info->ctx.is_scheduled) {
for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
if (!list_empty(
&kctx->jctx.sched_info.ctx.ctx_list_entry[js]))
list_del_init(
&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
}
/* The following events require us to kill off remaining jobs
* and update PM book-keeping:
* - we evicted it correctly (it must have jobs to be in the
* Policy Queue)
*
* These events need no action, but take this path anyway:
* - Case a: it didn't have any jobs, and was never in the Queue
* - Case b: scheduling transaction will be partially rolled-
* back (this already cancels the jobs)
*/
KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u,
js_kctx_info->ctx.is_scheduled);
dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx);
/* Only cancel jobs when we evicted from the policy
* queue. No Power Manager active reference was held.
*
* Having is_dying set ensures that this kills, and
* doesn't requeue */
kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
} else {
unsigned long flags;
bool was_retained;
/* Case c: didn't evict, but it is scheduled - it's in the Run
* Pool */
KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u,
js_kctx_info->ctx.is_scheduled);
dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx);
/* Disable the ctx from submitting any more jobs */
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
kbasep_js_clear_submit_allowed(js_devdata, kctx);
/* Retain and (later) release the context whilst it is is now
* disallowed from submitting jobs - ensures that someone
* somewhere will be removing the context later on */
was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
/* Since it's scheduled and we have the jsctx_mutex, it must be
* retained successfully */
KBASE_DEBUG_ASSERT(was_retained);
dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx);
/* Cancel any remaining running jobs for this kctx - if any.
* Submit is disallowed which takes effect immediately, so no
* more new jobs will appear after we do this. */
for (js = 0; js < kbdev->gpu_props.num_job_slots; js++)
kbase_job_slot_hardstop(kctx, js, NULL);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)",
kctx);
kbasep_js_runpool_release_ctx(kbdev, kctx);
}
KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u);
/* After this, you must wait on both the
* kbase_jd_context::zero_jobs_wait and the
* kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs
* to be destroyed, and the context to be de-scheduled (if it was on the
* runpoo