| /* |
| * |
| * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. |
| * |
| * This program is free software and is provided to you under the terms of the |
| * GNU General Public License version 2 as published by the Free Software |
| * Foundation, and any use by you of this program is subject to the terms |
| * of such GNU licence. |
| * |
| * A copy of the licence is included with the program, and can also be obtained |
| * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, |
| * Boston, MA 02110-1301, USA. |
| * |
| */ |
| |
| |
| |
| |
| /* |
| * Register-based HW access backend specific APIs |
| */ |
| |
| #include <mali_kbase.h> |
| #include <mali_kbase_hwaccess_jm.h> |
| #include <mali_kbase_jm.h> |
| #include <mali_kbase_js.h> |
| #include <mali_kbase_tlstream.h> |
| #include <mali_kbase_10969_workaround.h> |
| #include <backend/gpu/mali_kbase_cache_policy_backend.h> |
| #include <backend/gpu/mali_kbase_device_internal.h> |
| #include <backend/gpu/mali_kbase_jm_internal.h> |
| #include <backend/gpu/mali_kbase_js_affinity.h> |
| #include <backend/gpu/mali_kbase_pm_internal.h> |
| |
| /* Return whether the specified ringbuffer is empty. HW access lock must be |
| * held */ |
| #define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) |
| /* Return number of atoms currently in the specified ringbuffer. HW access lock |
| * must be held */ |
| #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) |
| |
| static void kbase_gpu_release_atom(struct kbase_device *kbdev, |
| struct kbase_jd_atom *katom, |
| ktime_t *end_timestamp); |
| |
| /** |
| * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer |
| * @kbdev: Device pointer |
| * @katom: Atom to enqueue |
| * |
| * Context: Caller must hold the HW access lock |
| */ |
| static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, |
| struct kbase_jd_atom *katom) |
| { |
| struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; |
| |
| WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; |
| rb->write_idx++; |
| |
| katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; |
| } |
| |
| /** |
| * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once |
| * it has been completed |
| * @kbdev: Device pointer |
| * @js: Job slot to remove atom from |
| * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in |
| * which case current time will be used. |
| * |
| * Context: Caller must hold the HW access lock |
| * |
| * Return: Atom removed from ringbuffer |
| */ |
| static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, |
| int js, |
| ktime_t *end_timestamp) |
| { |
| struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; |
| struct kbase_jd_atom *katom; |
| |
| if (SLOT_RB_EMPTY(rb)) { |
| WARN(1, "GPU ringbuffer unexpectedly empty\n"); |
| return NULL; |
| } |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; |
| |
| kbase_gpu_release_atom(kbdev, katom, end_timestamp); |
| |
| rb->read_idx++; |
| |
| katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; |
| |
| kbase_js_debug_log_current_affinities(kbdev); |
| |
| return katom; |
| } |
| |
| struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, |
| int idx) |
| { |
| struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| if ((SLOT_RB_ENTRIES(rb) - 1) < idx) |
| return NULL; /* idx out of range */ |
| |
| return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; |
| } |
| |
| struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, |
| int js) |
| { |
| return kbase_gpu_inspect(kbdev, js, 0); |
| } |
| |
| struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, |
| int js) |
| { |
| struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; |
| |
| if (SLOT_RB_EMPTY(rb)) |
| return NULL; |
| |
| return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; |
| } |
| |
| /** |
| * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently |
| * on the GPU |
| * @kbdev: Device pointer |
| * @js: Job slot to inspect |
| * |
| * Return: true if there are atoms on the GPU for slot js, |
| * false otherwise |
| */ |
| static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js) |
| { |
| int i; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| for (i = 0; i < SLOT_RB_SIZE; i++) { |
| struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); |
| |
| if (!katom) |
| return false; |
| if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED || |
| katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY) |
| return true; |
| } |
| |
| return false; |
| } |
| |
| /** |
| * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms |
| * currently on the GPU |
| * @kbdev: Device pointer |
| * |
| * Return: true if there are any atoms on the GPU, false otherwise |
| */ |
| static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) |
| { |
| int js; |
| int i; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { |
| for (i = 0; i < SLOT_RB_SIZE; i++) { |
| struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); |
| |
| if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js) |
| { |
| int nr = 0; |
| int i; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| for (i = 0; i < SLOT_RB_SIZE; i++) { |
| struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); |
| |
| if (katom && (katom->gpu_rb_state == |
| KBASE_ATOM_GPU_RB_SUBMITTED)) |
| nr++; |
| } |
| |
| return nr; |
| } |
| |
| int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js) |
| { |
| int nr = 0; |
| int i; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| for (i = 0; i < SLOT_RB_SIZE; i++) { |
| if (kbase_gpu_inspect(kbdev, js, i)) |
| nr++; |
| } |
| |
| return nr; |
| } |
| |
| static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js, |
| enum kbase_atom_gpu_rb_state min_rb_state) |
| { |
| int nr = 0; |
| int i; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| for (i = 0; i < SLOT_RB_SIZE; i++) { |
| struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); |
| |
| if (katom && (katom->gpu_rb_state >= min_rb_state)) |
| nr++; |
| } |
| |
| return nr; |
| } |
| |
| /** |
| * check_secure_atom - Check if the given atom is in the given secure state and |
| * has a ringbuffer state of at least |
| * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION |
| * @katom: Atom pointer |
| * @secure: Desired secure state |
| * |
| * Return: true if atom is in the given state, false otherwise |
| */ |
| static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) |
| { |
| if (katom->gpu_rb_state >= |
| KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && |
| ((kbase_jd_katom_is_protected(katom) && secure) || |
| (!kbase_jd_katom_is_protected(katom) && !secure))) |
| return true; |
| |
| return false; |
| } |
| |
| /** |
| * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given |
| * secure state in the ringbuffers of at least |
| * state |
| * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE |
| * @kbdev: Device pointer |
| * @secure: Desired secure state |
| * |
| * Return: true if any atoms are in the given state, false otherwise |
| */ |
| static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, |
| bool secure) |
| { |
| int js, i; |
| |
| for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { |
| for (i = 0; i < SLOT_RB_SIZE; i++) { |
| struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, |
| js, i); |
| |
| if (katom) { |
| if (check_secure_atom(katom, secure)) |
| return true; |
| } |
| } |
| } |
| |
| return false; |
| } |
| |
| int kbase_backend_slot_free(struct kbase_device *kbdev, int js) |
| { |
| if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != |
| KBASE_RESET_GPU_NOT_PENDING) { |
| /* The GPU is being reset - so prevent submission */ |
| return 0; |
| } |
| |
| return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); |
| } |
| |
| |
| static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, |
| struct kbase_jd_atom *katom); |
| |
| static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev, |
| int js, |
| struct kbase_jd_atom *katom) |
| { |
| /* The most recently checked affinity. Having this at this scope allows |
| * us to guarantee that we've checked the affinity in this function |
| * call. |
| */ |
| u64 recently_chosen_affinity = 0; |
| bool chosen_affinity = false; |
| bool retry; |
| |
| do { |
| retry = false; |
| |
| /* NOTE: The following uses a number of FALLTHROUGHs to optimize |
| * the calls to this function. Ending of the function is |
| * indicated by BREAK OUT */ |
| switch (katom->coreref_state) { |
| /* State when job is first attempted to be run */ |
| case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: |
| KBASE_DEBUG_ASSERT(katom->affinity == 0); |
| |
| /* Compute affinity */ |
| if (false == kbase_js_choose_affinity( |
| &recently_chosen_affinity, kbdev, katom, |
| js)) { |
| /* No cores are currently available */ |
| /* *** BREAK OUT: No state transition *** */ |
| break; |
| } |
| |
| chosen_affinity = true; |
| |
| /* Request the cores */ |
| kbase_pm_request_cores(kbdev, |
| katom->core_req & BASE_JD_REQ_T, |
| recently_chosen_affinity); |
| |
| katom->affinity = recently_chosen_affinity; |
| |
| /* Proceed to next state */ |
| katom->coreref_state = |
| KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: |
| { |
| enum kbase_pm_cores_ready cores_ready; |
| |
| KBASE_DEBUG_ASSERT(katom->affinity != 0 || |
| (katom->core_req & BASE_JD_REQ_T)); |
| |
| cores_ready = kbase_pm_register_inuse_cores( |
| kbdev, |
| katom->core_req & BASE_JD_REQ_T, |
| katom->affinity); |
| if (cores_ready == KBASE_NEW_AFFINITY) { |
| /* Affinity no longer valid - return to |
| * previous state */ |
| kbasep_js_job_check_deref_cores(kbdev, |
| katom); |
| KBASE_TRACE_ADD_SLOT_INFO(kbdev, |
| JS_CORE_REF_REGISTER_INUSE_FAILED, |
| katom->kctx, katom, |
| katom->jc, js, |
| (u32) katom->affinity); |
| /* *** BREAK OUT: Return to previous |
| * state, retry *** */ |
| retry = true; |
| break; |
| } |
| if (cores_ready == KBASE_CORES_NOT_READY) { |
| /* Stay in this state and return, to |
| * retry at this state later */ |
| KBASE_TRACE_ADD_SLOT_INFO(kbdev, |
| JS_CORE_REF_REGISTER_INUSE_FAILED, |
| katom->kctx, katom, |
| katom->jc, js, |
| (u32) katom->affinity); |
| /* *** BREAK OUT: No state transition |
| * *** */ |
| break; |
| } |
| /* Proceed to next state */ |
| katom->coreref_state = |
| KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; |
| } |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: |
| KBASE_DEBUG_ASSERT(katom->affinity != 0 || |
| (katom->core_req & BASE_JD_REQ_T)); |
| |
| /* Optimize out choosing the affinity twice in the same |
| * function call */ |
| if (chosen_affinity == false) { |
| /* See if the affinity changed since a previous |
| * call. */ |
| if (false == kbase_js_choose_affinity( |
| &recently_chosen_affinity, |
| kbdev, katom, js)) { |
| /* No cores are currently available */ |
| kbasep_js_job_check_deref_cores(kbdev, |
| katom); |
| KBASE_TRACE_ADD_SLOT_INFO(kbdev, |
| JS_CORE_REF_REQUEST_ON_RECHECK_FAILED, |
| katom->kctx, katom, |
| katom->jc, js, |
| (u32) recently_chosen_affinity); |
| /* *** BREAK OUT: Transition to lower |
| * state *** */ |
| break; |
| } |
| chosen_affinity = true; |
| } |
| |
| /* Now see if this requires a different set of cores */ |
| if (recently_chosen_affinity != katom->affinity) { |
| enum kbase_pm_cores_ready cores_ready; |
| |
| kbase_pm_request_cores(kbdev, |
| katom->core_req & BASE_JD_REQ_T, |
| recently_chosen_affinity); |
| |
| /* Register new cores whilst we still hold the |
| * old ones, to minimize power transitions */ |
| cores_ready = |
| kbase_pm_register_inuse_cores(kbdev, |
| katom->core_req & BASE_JD_REQ_T, |
| recently_chosen_affinity); |
| kbasep_js_job_check_deref_cores(kbdev, katom); |
| |
| /* Fixup the state that was reduced by |
| * deref_cores: */ |
| katom->coreref_state = |
| KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; |
| katom->affinity = recently_chosen_affinity; |
| if (cores_ready == KBASE_NEW_AFFINITY) { |
| /* Affinity no longer valid - return to |
| * previous state */ |
| katom->coreref_state = |
| KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; |
| |
| kbasep_js_job_check_deref_cores(kbdev, |
| katom); |
| |
| KBASE_TRACE_ADD_SLOT_INFO(kbdev, |
| JS_CORE_REF_REGISTER_INUSE_FAILED, |
| katom->kctx, katom, |
| katom->jc, js, |
| (u32) katom->affinity); |
| /* *** BREAK OUT: Return to previous |
| * state, retry *** */ |
| retry = true; |
| break; |
| } |
| /* Now might be waiting for powerup again, with |
| * a new affinity */ |
| if (cores_ready == KBASE_CORES_NOT_READY) { |
| /* Return to previous state */ |
| katom->coreref_state = |
| KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; |
| KBASE_TRACE_ADD_SLOT_INFO(kbdev, |
| JS_CORE_REF_REGISTER_ON_RECHECK_FAILED, |
| katom->kctx, katom, |
| katom->jc, js, |
| (u32) katom->affinity); |
| /* *** BREAK OUT: Transition to lower |
| * state *** */ |
| break; |
| } |
| } |
| /* Proceed to next state */ |
| katom->coreref_state = |
| KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS: |
| KBASE_DEBUG_ASSERT(katom->affinity != 0 || |
| (katom->core_req & BASE_JD_REQ_T)); |
| KBASE_DEBUG_ASSERT(katom->affinity == |
| recently_chosen_affinity); |
| |
| /* Note: this is where the caller must've taken the |
| * hwaccess_lock */ |
| |
| /* Check for affinity violations - if there are any, |
| * then we just ask the caller to requeue and try again |
| * later */ |
| if (kbase_js_affinity_would_violate(kbdev, js, |
| katom->affinity) != false) { |
| /* Return to previous state */ |
| katom->coreref_state = |
| KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; |
| /* *** BREAK OUT: Transition to lower state *** |
| */ |
| KBASE_TRACE_ADD_SLOT_INFO(kbdev, |
| JS_CORE_REF_AFFINITY_WOULD_VIOLATE, |
| katom->kctx, katom, katom->jc, js, |
| (u32) katom->affinity); |
| break; |
| } |
| |
| /* No affinity violations would result, so the cores are |
| * ready */ |
| katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY; |
| /* *** BREAK OUT: Cores Ready *** */ |
| break; |
| |
| default: |
| KBASE_DEBUG_ASSERT_MSG(false, |
| "Unhandled kbase_atom_coreref_state %d", |
| katom->coreref_state); |
| break; |
| } |
| } while (retry != false); |
| |
| return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY); |
| } |
| |
| static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, |
| struct kbase_jd_atom *katom) |
| { |
| KBASE_DEBUG_ASSERT(kbdev != NULL); |
| KBASE_DEBUG_ASSERT(katom != NULL); |
| |
| switch (katom->coreref_state) { |
| case KBASE_ATOM_COREREF_STATE_READY: |
| /* State where atom was submitted to the HW - just proceed to |
| * power-down */ |
| KBASE_DEBUG_ASSERT(katom->affinity != 0 || |
| (katom->core_req & BASE_JD_REQ_T)); |
| |
| /* *** FALLTHROUGH *** */ |
| |
| case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: |
| /* State where cores were registered */ |
| KBASE_DEBUG_ASSERT(katom->affinity != 0 || |
| (katom->core_req & BASE_JD_REQ_T)); |
| kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T, |
| katom->affinity); |
| |
| break; |
| |
| case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: |
| /* State where cores were requested, but not registered */ |
| KBASE_DEBUG_ASSERT(katom->affinity != 0 || |
| (katom->core_req & BASE_JD_REQ_T)); |
| kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T, |
| katom->affinity); |
| break; |
| |
| case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: |
| /* Initial state - nothing required */ |
| KBASE_DEBUG_ASSERT(katom->affinity == 0); |
| break; |
| |
| default: |
| KBASE_DEBUG_ASSERT_MSG(false, |
| "Unhandled coreref_state: %d", |
| katom->coreref_state); |
| break; |
| } |
| |
| katom->affinity = 0; |
| katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; |
| } |
| |
| static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev, |
| base_jd_core_req core_req, u64 affinity, |
| enum kbase_atom_coreref_state coreref_state) |
| { |
| KBASE_DEBUG_ASSERT(kbdev != NULL); |
| |
| switch (coreref_state) { |
| case KBASE_ATOM_COREREF_STATE_READY: |
| /* State where atom was submitted to the HW - just proceed to |
| * power-down */ |
| KBASE_DEBUG_ASSERT(affinity != 0 || |
| (core_req & BASE_JD_REQ_T)); |
| |
| /* *** FALLTHROUGH *** */ |
| |
| case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: |
| /* State where cores were registered */ |
| KBASE_DEBUG_ASSERT(affinity != 0 || |
| (core_req & BASE_JD_REQ_T)); |
| kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T, |
| affinity); |
| |
| break; |
| |
| case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: |
| /* State where cores were requested, but not registered */ |
| KBASE_DEBUG_ASSERT(affinity != 0 || |
| (core_req & BASE_JD_REQ_T)); |
| kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T, |
| affinity); |
| break; |
| |
| case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: |
| /* Initial state - nothing required */ |
| KBASE_DEBUG_ASSERT(affinity == 0); |
| break; |
| |
| default: |
| KBASE_DEBUG_ASSERT_MSG(false, |
| "Unhandled coreref_state: %d", |
| coreref_state); |
| break; |
| } |
| } |
| |
| static void kbase_gpu_release_atom(struct kbase_device *kbdev, |
| struct kbase_jd_atom *katom, |
| ktime_t *end_timestamp) |
| { |
| switch (katom->gpu_rb_state) { |
| case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: |
| /* Should be impossible */ |
| WARN(1, "Attempting to release atom not in ringbuffer\n"); |
| break; |
| |
| case KBASE_ATOM_GPU_RB_SUBMITTED: |
| /* Inform power management at start/finish of atom so it can |
| * update its GPU utilisation metrics. Mark atom as not |
| * submitted beforehand. */ |
| katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; |
| kbase_pm_metrics_update(kbdev, end_timestamp); |
| |
| if (katom->core_req & BASE_JD_REQ_PERMON) |
| kbase_pm_release_gpu_cycle_counter_nolock(kbdev); |
| /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_READY: |
| /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: |
| kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr, |
| katom->affinity); |
| /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: |
| break; |
| |
| case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: |
| if (katom->protected_state.enter != |
| KBASE_ATOM_ENTER_PROTECTED_CHECK || |
| katom->protected_state.exit != |
| KBASE_ATOM_EXIT_PROTECTED_CHECK) |
| kbdev->protected_mode_transition = false; |
| |
| if (kbase_jd_katom_is_protected(katom) && |
| (katom->protected_state.enter == |
| KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) |
| kbase_vinstr_resume(kbdev->vinstr_ctx); |
| |
| /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: |
| /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: |
| /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_RETURN_TO_JS: |
| break; |
| } |
| |
| katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; |
| } |
| |
| static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, |
| struct kbase_jd_atom *katom) |
| { |
| kbase_gpu_release_atom(kbdev, katom, NULL); |
| katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; |
| } |
| |
| static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js) |
| { |
| struct kbase_backend_data *backend = &kbdev->hwaccess.backend; |
| bool slot_busy[3]; |
| |
| if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) |
| return true; |
| slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0, |
| KBASE_ATOM_GPU_RB_WAITING_AFFINITY); |
| slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1, |
| KBASE_ATOM_GPU_RB_WAITING_AFFINITY); |
| slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2, |
| KBASE_ATOM_GPU_RB_WAITING_AFFINITY); |
| |
| if ((js == 2 && !(slot_busy[0] || slot_busy[1])) || |
| (js != 2 && !slot_busy[2])) |
| return true; |
| |
| /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */ |
| if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) || |
| kbase_gpu_atoms_submitted(kbdev, 1) || |
| backend->rmu_workaround_flag)) |
| return false; |
| |
| /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */ |
| if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) || |
| !backend->rmu_workaround_flag)) |
| return false; |
| |
| backend->rmu_workaround_flag = !backend->rmu_workaround_flag; |
| |
| return true; |
| } |
| |
| static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) |
| { |
| return kbdev->protected_mode; |
| } |
| |
| static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) |
| { |
| int err = -EINVAL; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| WARN_ONCE(!kbdev->protected_ops, |
| "Cannot enter protected mode: protected callbacks not specified.\n"); |
| |
| /* |
| * When entering into protected mode, we must ensure that the |
| * GPU is not operating in coherent mode as well. This is to |
| * ensure that no protected memory can be leaked. |
| */ |
| if (kbdev->system_coherency == COHERENCY_ACE) |
| kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); |
| |
| if (kbdev->protected_ops) { |
| /* Switch GPU to protected mode */ |
| err = kbdev->protected_ops->protected_mode_enter(kbdev); |
| |
| if (err) |
| dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", |
| err); |
| else |
| kbdev->protected_mode = true; |
| } |
| |
| return err; |
| } |
| |
| static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) |
| { |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| WARN_ONCE(!kbdev->protected_ops, |
| "Cannot exit protected mode: protected callbacks not specified.\n"); |
| |
| if (!kbdev->protected_ops) |
| return -EINVAL; |
| |
| kbase_reset_gpu_silent(kbdev); |
| |
| return 0; |
| } |
| |
| static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, |
| struct kbase_jd_atom **katom, int idx, int js) |
| { |
| int err = 0; |
| |
| switch (katom[idx]->protected_state.enter) { |
| case KBASE_ATOM_ENTER_PROTECTED_CHECK: |
| /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV |
| * should ensure that we are not already transitiong, and that |
| * there are no atoms currently on the GPU. */ |
| WARN_ON(kbdev->protected_mode_transition); |
| WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); |
| |
| kbdev->protected_mode_transition = true; |
| katom[idx]->protected_state.enter = |
| KBASE_ATOM_ENTER_PROTECTED_VINSTR; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_ENTER_PROTECTED_VINSTR: |
| if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) { |
| /* |
| * We can't switch now because |
| * the vinstr core state switch |
| * is not done yet. |
| */ |
| return -EAGAIN; |
| } |
| |
| /* Once reaching this point GPU must be |
| * switched to protected mode or vinstr |
| * re-enabled. */ |
| |
| /* |
| * Not in correct mode, begin protected mode switch. |
| * Entering protected mode requires us to power down the L2, |
| * and drop out of fully coherent mode. |
| */ |
| katom[idx]->protected_state.enter = |
| KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; |
| |
| kbase_pm_update_cores_state_nolock(kbdev); |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: |
| /* Avoid unnecessary waiting on non-ACE platforms. */ |
| if (kbdev->current_gpu_coherency_mode == COHERENCY_ACE) { |
| if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || |
| kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { |
| /* |
| * The L2 is still powered, wait for all the users to |
| * finish with it before doing the actual reset. |
| */ |
| return -EAGAIN; |
| } |
| } |
| |
| katom[idx]->protected_state.enter = |
| KBASE_ATOM_ENTER_PROTECTED_FINISHED; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_ENTER_PROTECTED_FINISHED: |
| |
| /* No jobs running, so we can switch GPU mode right now. */ |
| err = kbase_gpu_protected_mode_enter(kbdev); |
| |
| /* |
| * Regardless of result, we are no longer transitioning |
| * the GPU. |
| */ |
| kbdev->protected_mode_transition = false; |
| |
| if (err) { |
| /* |
| * Failed to switch into protected mode, resume |
| * vinstr core and fail atom. |
| */ |
| kbase_vinstr_resume(kbdev->vinstr_ctx); |
| katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; |
| kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); |
| /* Only return if head atom or previous atom |
| * already removed - as atoms must be returned |
| * in order. */ |
| if (idx == 0 || katom[0]->gpu_rb_state == |
| KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { |
| kbase_gpu_dequeue_atom(kbdev, js, NULL); |
| kbase_jm_return_atom_to_js(kbdev, katom[idx]); |
| } |
| return -EINVAL; |
| } |
| |
| /* Protected mode sanity checks. */ |
| KBASE_DEBUG_ASSERT_MSG( |
| kbase_jd_katom_is_protected(katom[idx]) == |
| kbase_gpu_in_protected_mode(kbdev), |
| "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", |
| kbase_jd_katom_is_protected(katom[idx]), |
| kbase_gpu_in_protected_mode(kbdev)); |
| katom[idx]->gpu_rb_state = |
| KBASE_ATOM_GPU_RB_READY; |
| } |
| |
| return 0; |
| } |
| |
| static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, |
| struct kbase_jd_atom **katom, int idx, int js) |
| { |
| int err = 0; |
| |
| |
| switch (katom[idx]->protected_state.exit) { |
| case KBASE_ATOM_EXIT_PROTECTED_CHECK: |
| /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV |
| * should ensure that we are not already transitiong, and that |
| * there are no atoms currently on the GPU. */ |
| WARN_ON(kbdev->protected_mode_transition); |
| WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); |
| |
| /* |
| * Exiting protected mode requires a reset, but first the L2 |
| * needs to be powered down to ensure it's not active when the |
| * reset is issued. |
| */ |
| katom[idx]->protected_state.exit = |
| KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; |
| |
| kbdev->protected_mode_transition = true; |
| kbase_pm_update_cores_state_nolock(kbdev); |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: |
| if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || |
| kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { |
| /* |
| * The L2 is still powered, wait for all the users to |
| * finish with it before doing the actual reset. |
| */ |
| return -EAGAIN; |
| } |
| katom[idx]->protected_state.exit = |
| KBASE_ATOM_EXIT_PROTECTED_RESET; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_EXIT_PROTECTED_RESET: |
| /* Issue the reset to the GPU */ |
| err = kbase_gpu_protected_mode_reset(kbdev); |
| |
| if (err) { |
| kbdev->protected_mode_transition = false; |
| |
| /* Failed to exit protected mode, fail atom */ |
| katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; |
| kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); |
| /* Only return if head atom or previous atom |
| * already removed - as atoms must be returned |
| * in order */ |
| if (idx == 0 || katom[0]->gpu_rb_state == |
| KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { |
| kbase_gpu_dequeue_atom(kbdev, js, NULL); |
| kbase_jm_return_atom_to_js(kbdev, katom[idx]); |
| } |
| |
| kbase_vinstr_resume(kbdev->vinstr_ctx); |
| |
| return -EINVAL; |
| } |
| |
| katom[idx]->protected_state.exit = |
| KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: |
| if (kbase_reset_gpu_active(kbdev)) |
| return -EAGAIN; |
| |
| kbdev->protected_mode_transition = false; |
| kbdev->protected_mode = false; |
| |
| /* protected mode sanity checks */ |
| KBASE_DEBUG_ASSERT_MSG( |
| kbase_jd_katom_is_protected(katom[idx]) == kbase_gpu_in_protected_mode(kbdev), |
| "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", |
| kbase_jd_katom_is_protected(katom[idx]), kbase_gpu_in_protected_mode(kbdev)); |
| KBASE_DEBUG_ASSERT_MSG( |
| (kbase_jd_katom_is_protected(katom[idx]) && js == 0) || |
| !kbase_jd_katom_is_protected(katom[idx]), |
| "Protected atom on JS%d not supported", js); |
| } |
| |
| return 0; |
| } |
| |
| void kbase_backend_slot_update(struct kbase_device *kbdev) |
| { |
| int js; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { |
| struct kbase_jd_atom *katom[2]; |
| int idx; |
| |
| katom[0] = kbase_gpu_inspect(kbdev, js, 0); |
| katom[1] = kbase_gpu_inspect(kbdev, js, 1); |
| WARN_ON(katom[1] && !katom[0]); |
| |
| for (idx = 0; idx < SLOT_RB_SIZE; idx++) { |
| bool cores_ready; |
| int ret; |
| |
| if (!katom[idx]) |
| continue; |
| |
| switch (katom[idx]->gpu_rb_state) { |
| case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: |
| /* Should be impossible */ |
| WARN(1, "Attempting to update atom not in ringbuffer\n"); |
| break; |
| |
| case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: |
| if (katom[idx]->atom_flags & |
| KBASE_KATOM_FLAG_X_DEP_BLOCKED) |
| break; |
| |
| katom[idx]->gpu_rb_state = |
| KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: |
| if (kbase_gpu_check_secure_atoms(kbdev, |
| !kbase_jd_katom_is_protected( |
| katom[idx]))) |
| break; |
| |
| if (kbdev->protected_mode_transition) |
| break; |
| |
| katom[idx]->gpu_rb_state = |
| KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: |
| |
| /* |
| * Exiting protected mode must be done before |
| * the references on the cores are taken as |
| * a power down the L2 is required which |
| * can't happen after the references for this |
| * atom are taken. |
| */ |
| |
| if (!kbase_gpu_in_protected_mode(kbdev) && |
| kbase_jd_katom_is_protected(katom[idx])) { |
| /* Atom needs to transition into protected mode. */ |
| ret = kbase_jm_enter_protected_mode(kbdev, |
| katom, idx, js); |
| if (ret) |
| break; |
| } else if (kbase_gpu_in_protected_mode(kbdev) && |
| !kbase_jd_katom_is_protected(katom[idx])) { |
| /* Atom needs to transition out of protected mode. */ |
| ret = kbase_jm_exit_protected_mode(kbdev, |
| katom, idx, js); |
| if (ret) |
| break; |
| } |
| katom[idx]->protected_state.exit = |
| KBASE_ATOM_EXIT_PROTECTED_CHECK; |
| |
| /* Atom needs no protected mode transition. */ |
| |
| katom[idx]->gpu_rb_state = |
| KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: |
| if (katom[idx]->will_fail_event_code) { |
| kbase_gpu_mark_atom_for_return(kbdev, |
| katom[idx]); |
| /* Set EVENT_DONE so this atom will be |
| completed, not unpulled. */ |
| katom[idx]->event_code = |
| BASE_JD_EVENT_DONE; |
| /* Only return if head atom or previous |
| * atom already removed - as atoms must |
| * be returned in order. */ |
| if (idx == 0 || katom[0]->gpu_rb_state == |
| KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { |
| kbase_gpu_dequeue_atom(kbdev, js, NULL); |
| kbase_jm_return_atom_to_js(kbdev, katom[idx]); |
| } |
| break; |
| } |
| |
| cores_ready = |
| kbasep_js_job_check_ref_cores(kbdev, js, |
| katom[idx]); |
| |
| if (katom[idx]->event_code == |
| BASE_JD_EVENT_PM_EVENT) { |
| katom[idx]->gpu_rb_state = |
| KBASE_ATOM_GPU_RB_RETURN_TO_JS; |
| break; |
| } |
| |
| if (!cores_ready) |
| break; |
| |
| kbase_js_affinity_retain_slot_cores(kbdev, js, |
| katom[idx]->affinity); |
| katom[idx]->gpu_rb_state = |
| KBASE_ATOM_GPU_RB_WAITING_AFFINITY; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: |
| if (!kbase_gpu_rmu_workaround(kbdev, js)) |
| break; |
| |
| katom[idx]->gpu_rb_state = |
| KBASE_ATOM_GPU_RB_READY; |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_READY: |
| |
| /* Only submit if head atom or previous atom |
| * already submitted */ |
| if (idx == 1 && |
| (katom[0]->gpu_rb_state != |
| KBASE_ATOM_GPU_RB_SUBMITTED && |
| katom[0]->gpu_rb_state != |
| KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) |
| break; |
| |
| /* Check if this job needs the cycle counter |
| * enabled before submission */ |
| if (katom[idx]->core_req & BASE_JD_REQ_PERMON) |
| kbase_pm_request_gpu_cycle_counter_l2_is_on( |
| kbdev); |
| |
| kbase_job_hw_submit(kbdev, katom[idx], js); |
| katom[idx]->gpu_rb_state = |
| KBASE_ATOM_GPU_RB_SUBMITTED; |
| |
| /* Inform power management at start/finish of |
| * atom so it can update its GPU utilisation |
| * metrics. */ |
| kbase_pm_metrics_update(kbdev, |
| &katom[idx]->start_timestamp); |
| |
| /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ |
| |
| case KBASE_ATOM_GPU_RB_SUBMITTED: |
| /* Atom submitted to HW, nothing else to do */ |
| break; |
| |
| case KBASE_ATOM_GPU_RB_RETURN_TO_JS: |
| /* Only return if head atom or previous atom |
| * already removed - as atoms must be returned |
| * in order */ |
| if (idx == 0 || katom[0]->gpu_rb_state == |
| KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { |
| kbase_gpu_dequeue_atom(kbdev, js, NULL); |
| kbase_jm_return_atom_to_js(kbdev, |
| katom[idx]); |
| } |
| break; |
| } |
| } |
| } |
| |
| /* Warn if PRLAM-8987 affinity restrictions are violated */ |
| if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) |
| WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) || |
| kbase_gpu_atoms_submitted(kbdev, 1)) && |
| kbase_gpu_atoms_submitted(kbdev, 2)); |
| } |
| |
| |
| void kbase_backend_run_atom(struct kbase_device *kbdev, |
| struct kbase_jd_atom *katom) |
| { |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| kbase_gpu_enqueue_atom(kbdev, katom); |
| kbase_backend_slot_update(kbdev); |
| } |
| |
| bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js) |
| { |
| struct kbase_jd_atom *katom; |
| struct kbase_jd_atom *next_katom; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| katom = kbase_gpu_inspect(kbdev, js, 0); |
| next_katom = kbase_gpu_inspect(kbdev, js, 1); |
| |
| if (next_katom && katom->kctx == next_katom->kctx && |
| next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && |
| (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) |
| != 0 || |
| kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) |
| != 0)) { |
| kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), |
| JS_COMMAND_NOP, NULL); |
| next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; |
| return true; |
| } |
| |
| return false; |
| } |
| |
| void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, |
| u32 completion_code, |
| u64 job_tail, |
| ktime_t *end_timestamp) |
| { |
| struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); |
| struct kbase_context *kctx = katom->kctx; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) && |
| completion_code != BASE_JD_EVENT_DONE && |
| !(completion_code & BASE_JD_SW_EVENT)) { |
| katom->need_cache_flush_cores_retained = katom->affinity; |
| kbase_pm_request_cores(kbdev, false, katom->affinity); |
| } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) { |
| if (kbdev->gpu_props.num_core_groups > 1 && |
| !(katom->affinity & |
| kbdev->gpu_props.props.coherency_info.group[0].core_mask |
| ) && |
| (katom->affinity & |
| kbdev->gpu_props.props.coherency_info.group[1].core_mask |
| )) { |
| dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n"); |
| katom->need_cache_flush_cores_retained = |
| katom->affinity; |
| kbase_pm_request_cores(kbdev, false, |
| katom->affinity); |
| } |
| } |
| |
| katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); |
| kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0); |
| kbase_tlstream_tl_nret_atom_lpu( |
| katom, |
| &kbdev->gpu_props.props.raw_props.js_features[ |
| katom->slot_nr]); |
| kbase_tlstream_tl_nret_atom_as(katom, &kbdev->as[kctx->as_nr]); |
| kbase_tlstream_tl_nret_ctx_lpu( |
| kctx, |
| &kbdev->gpu_props.props.raw_props.js_features[ |
| katom->slot_nr]); |
| |
| if (completion_code == BASE_JD_EVENT_STOPPED) { |
| struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, |
| 0); |
| |
| /* |
| * Dequeue next atom from ringbuffers on same slot if required. |
| * This atom will already have been removed from the NEXT |
| * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that |
| * the atoms on this slot are returned in the correct order. |
| */ |
| if (next_katom && katom->kctx == next_katom->kctx) { |
| kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); |
| kbase_jm_return_atom_to_js(kbdev, next_katom); |
| } |
| } else if (completion_code != BASE_JD_EVENT_DONE) { |
| struct kbasep_js_device_data *js_devdata = &kbdev->js_data; |
| int i; |
| |
| #if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0 |
| KBASE_TRACE_DUMP(kbdev); |
| #endif |
| kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); |
| |
| /* |
| * Remove all atoms on the same context from ringbuffers. This |
| * will not remove atoms that are already on the GPU, as these |
| * are guaranteed not to have fail dependencies on the failed |
| * atom. |
| */ |
| for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { |
| struct kbase_jd_atom *katom_idx0 = |
| kbase_gpu_inspect(kbdev, i, 0); |
| struct kbase_jd_atom *katom_idx1 = |
| kbase_gpu_inspect(kbdev, i, 1); |
| |
| if (katom_idx0 && katom_idx0->kctx == katom->kctx && |
| katom_idx0->gpu_rb_state != |
| KBASE_ATOM_GPU_RB_SUBMITTED) { |
| /* Dequeue katom_idx0 from ringbuffer */ |
| kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); |
| |
| if (katom_idx1 && |
| katom_idx1->kctx == katom->kctx && |
| katom_idx0->gpu_rb_state != |
| KBASE_ATOM_GPU_RB_SUBMITTED) { |
| /* Dequeue katom_idx1 from ringbuffer */ |
| kbase_gpu_dequeue_atom(kbdev, i, |
| end_timestamp); |
| |
| katom_idx1->event_code = |
| BASE_JD_EVENT_STOPPED; |
| kbase_jm_return_atom_to_js(kbdev, |
| katom_idx1); |
| } |
| katom_idx0->event_code = BASE_JD_EVENT_STOPPED; |
| kbase_jm_return_atom_to_js(kbdev, katom_idx0); |
| |
| } else if (katom_idx1 && |
| katom_idx1->kctx == katom->kctx && |
| katom_idx1->gpu_rb_state != |
| KBASE_ATOM_GPU_RB_SUBMITTED) { |
| /* Can not dequeue this atom yet - will be |
| * dequeued when atom at idx0 completes */ |
| katom_idx1->event_code = BASE_JD_EVENT_STOPPED; |
| kbase_gpu_mark_atom_for_return(kbdev, |
| katom_idx1); |
| } |
| } |
| } |
| |
| KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, |
| js, completion_code); |
| |
| if (job_tail != 0 && job_tail != katom->jc) { |
| bool was_updated = (job_tail != katom->jc); |
| |
| /* Some of the job has been executed, so we update the job chain |
| * address to where we should resume from */ |
| katom->jc = job_tail; |
| if (was_updated) |
| KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, |
| katom, job_tail, js); |
| } |
| |
| /* Only update the event code for jobs that weren't cancelled */ |
| if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) |
| katom->event_code = (base_jd_event_code)completion_code; |
| |
| kbase_device_trace_register_access(kctx, REG_WRITE, |
| JOB_CONTROL_REG(JOB_IRQ_CLEAR), |
| 1 << js); |
| |
| /* Complete the job, and start new ones |
| * |
| * Also defer remaining work onto the workqueue: |
| * - Re-queue Soft-stopped jobs |
| * - For any other jobs, queue the job back into the dependency system |
| * - Schedule out the parent context if necessary, and schedule a new |
| * one in. |
| */ |
| #ifdef CONFIG_GPU_TRACEPOINTS |
| { |
| /* The atom in the HEAD */ |
| struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, |
| 0); |
| |
| if (next_katom && next_katom->gpu_rb_state == |
| KBASE_ATOM_GPU_RB_SUBMITTED) { |
| char js_string[16]; |
| |
| trace_gpu_sched_switch(kbasep_make_job_slot_string(js, |
| js_string), |
| ktime_to_ns(*end_timestamp), |
| (u32)next_katom->kctx->id, 0, |
| next_katom->work_id); |
| kbdev->hwaccess.backend.slot_rb[js].last_context = |
| next_katom->kctx; |
| } else { |
| char js_string[16]; |
| |
| trace_gpu_sched_switch(kbasep_make_job_slot_string(js, |
| js_string), |
| ktime_to_ns(ktime_get()), 0, 0, |
| 0); |
| kbdev->hwaccess.backend.slot_rb[js].last_context = 0; |
| } |
| } |
| #endif |
| |
| if (completion_code == BASE_JD_EVENT_STOPPED) |
| katom = kbase_jm_return_atom_to_js(kbdev, katom); |
| else |
| katom = kbase_jm_complete(kbdev, katom, end_timestamp); |
| |
| if (katom) { |
| /* Cross-slot dependency has now become runnable. Try to submit |
| * it. */ |
| |
| /* Check if there are lower priority jobs to soft stop */ |
| kbase_job_slot_ctx_priority_check_locked(kctx, katom); |
| |
| kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); |
| } |
| |
| /* Job completion may have unblocked other atoms. Try to update all job |
| * slots */ |
| kbase_backend_slot_update(kbdev); |
| } |
| |
| void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) |
| { |
| int js; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { |
| int atom_idx = 0; |
| int idx; |
| |
| for (idx = 0; idx < SLOT_RB_SIZE; idx++) { |
| struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, |
| js, atom_idx); |
| bool keep_in_jm_rb = false; |
| |
| if (!katom) |
| break; |
| |
| if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) |
| keep_in_jm_rb = true; |
| |
| kbase_gpu_release_atom(kbdev, katom, NULL); |
| |
| /* |
| * If the atom wasn't on HW when the reset was issued |
| * then leave it in the RB and next time we're kicked |
| * it will be processed again from the starting state. |
| */ |
| if (keep_in_jm_rb) { |
| katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; |
| katom->affinity = 0; |
| katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; |
| /* As the atom was not removed, increment the |
| * index so that we read the correct atom in the |
| * next iteration. */ |
| atom_idx++; |
| continue; |
| } |
| |
| /* |
| * The atom was on the HW when the reset was issued |
| * all we can do is fail the atom. |
| */ |
| kbase_gpu_dequeue_atom(kbdev, js, NULL); |
| katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; |
| kbase_jm_complete(kbdev, katom, end_timestamp); |
| } |
| } |
| |
| kbdev->protected_mode_transition = false; |
| kbdev->protected_mode = false; |
| } |
| |
| static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, |
| int js, |
| struct kbase_jd_atom *katom, |
| u32 action) |
| { |
| struct kbasep_js_device_data *js_devdata = &kbdev->js_data; |
| u32 hw_action = action & JS_COMMAND_MASK; |
| |
| kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); |
| kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, |
| katom->core_req, katom); |
| kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); |
| } |
| |
| static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, |
| struct kbase_jd_atom *katom, |
| u32 action, |
| bool disjoint) |
| { |
| struct kbasep_js_device_data *js_devdata = &kbdev->js_data; |
| |
| katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; |
| kbase_gpu_mark_atom_for_return(kbdev, katom); |
| kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); |
| |
| if (disjoint) |
| kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, |
| katom); |
| } |
| |
| static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) |
| { |
| if (katom->x_post_dep) { |
| struct kbase_jd_atom *dep_atom = katom->x_post_dep; |
| |
| if (dep_atom->gpu_rb_state != |
| KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && |
| dep_atom->gpu_rb_state != |
| KBASE_ATOM_GPU_RB_RETURN_TO_JS) |
| return dep_atom->slot_nr; |
| } |
| return -1; |
| } |
| |
| static void kbase_job_evicted(struct kbase_jd_atom *katom) |
| { |
| kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom, |
| katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT); |
| } |
| |
| bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, |
| struct kbase_context *kctx, |
| int js, |
| struct kbase_jd_atom *katom, |
| u32 action) |
| { |
| struct kbasep_js_device_data *js_devdata = &kbdev->js_data; |
| |
| struct kbase_jd_atom *katom_idx0; |
| struct kbase_jd_atom *katom_idx1; |
| |
| bool katom_idx0_valid, katom_idx1_valid; |
| |
| bool ret = false; |
| |
| int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; |
| |
| lockdep_assert_held(&kbdev->hwaccess_lock); |
| |
| katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); |
| katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); |
| |
| if (katom) { |
| katom_idx0_valid = (katom_idx0 == katom); |
| /* If idx0 is to be removed and idx1 is on the same context, |
| * then idx1 must also be removed otherwise the atoms might be |
| * returned out of order */ |
| if (katom_idx1) |
| katom_idx1_valid = (katom_idx1 == katom) || |
| (katom_idx0_valid && |
| (katom_idx0->kctx == |
| katom_idx1->kctx)); |
| else |
| katom_idx1_valid = false; |
| } else { |
| katom_idx0_valid = (katom_idx0 && |
| (!kctx || katom_idx0->kctx == kctx)); |
| katom_idx1_valid = (katom_idx1 && |
| (!kctx || katom_idx1->kctx == kctx)); |
| } |
| |
| if (katom_idx0_valid) |
| stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); |
| if (katom_idx1_valid) |
| stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); |
| |
| if (katom_idx0_valid) { |
| if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { |
| /* Simple case - just dequeue and return */ |
| kbase_gpu_dequeue_atom(kbdev, js, NULL); |
| if (katom_idx1_valid) { |
| kbase_gpu_dequeue_atom(kbdev, js, NULL); |
| katom_idx1->event_code = |
| BASE_JD_EVENT_REMOVED_FROM_NEXT; |
| kbase_jm_return_atom_to_js(kbdev, katom_idx1); |
| kbasep_js_clear_submit_allowed(js_devdata, |
| katom_idx1->kctx); |
| } |
| |
| katom_idx0->event_code = |
| BASE_JD_EVENT_REMOVED_FROM_NEXT; |
| kbase_jm_return_atom_to_js(kbdev, katom_idx0); |
| kbasep_js_clear_submit_allowed(js_devdata, |
| katom_idx0->kctx); |
| } else { |
| /* katom_idx0 is on GPU */ |
| if (katom_idx1 && katom_idx1->gpu_rb_state == |
| KBASE_ATOM_GPU_RB_SUBMITTED) { |
| /* katom_idx0 and katom_idx1 are on GPU */ |
| |
| if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, |
| JS_COMMAND_NEXT), NULL) == 0) { |
| /* idx0 has already completed - stop |
| * idx1 if needed*/ |
| if (katom_idx1_valid) { |
| kbase_gpu_stop_atom(kbdev, js, |
| katom_idx1, |
| action); |
| ret = true; |
| } |
| } else { |
| /* idx1 is in NEXT registers - attempt |
| * to remove */ |
| kbase_reg_write(kbdev, |
| JOB_SLOT_REG(js, |
| JS_COMMAND_NEXT), |
| JS_COMMAND_NOP, NULL); |
| |
| if (kbase_reg_read(kbdev, |
| JOB_SLOT_REG(js, |
| JS_HEAD_NEXT_LO), NULL) |
| != 0 || |
| kbase_reg_read(kbdev, |
| JOB_SLOT_REG(js, |
| JS_HEAD_NEXT_HI), NULL) |
| != 0) { |
| /* idx1 removed successfully, |
| * will be handled in IRQ */ |
| kbase_job_evicted(katom_idx1); |
| kbase_gpu_remove_atom(kbdev, |
| katom_idx1, |
| action, true); |
| stop_x_dep_idx1 = |
| should_stop_x_dep_slot(katom_idx1); |
| |
| /* stop idx0 if still on GPU */ |
| kbase_gpu_stop_atom(kbdev, js, |
| katom_idx0, |
| action); |
| ret = true; |
| } else if (katom_idx1_valid) { |
| /* idx0 has already completed, |
| * stop idx1 if needed */ |
| kbase_gpu_stop_atom(kbdev, js, |
| katom_idx1, |
| action); |
| ret = true; |
| } |
| } |
| } else if (katom_idx1_valid) { |
| /* idx1 not on GPU but must be dequeued*/ |
| |
| /* idx1 will be handled in IRQ */ |
| kbase_gpu_remove_atom(kbdev, katom_idx1, action, |
| false); |
| /* stop idx0 */ |
| /* This will be repeated for anything removed |
| * from the next registers, since their normal |
| * flow was also interrupted, and this function |
| * might not enter disjoint state e.g. if we |
| * don't actually do a hard stop on the head |
| * atom */ |
| kbase_gpu_stop_atom(kbdev, js, katom_idx0, |
| action); |
| ret = true; |
| } else { |
| /* no atom in idx1 */ |
| /* just stop idx0 */ |
| kbase_gpu_stop_atom(kbdev, js, katom_idx0, |
| action); |
| ret = true; |
| } |
| } |
| } else if (katom_idx1_valid) { |
| if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { |
| /* Mark for return */ |
| /* idx1 will be returned once idx0 completes */ |
| kbase_gpu_remove_atom(kbdev, katom_idx1, action, |
| false); |
| } else { |
| /* idx1 is on GPU */ |
| if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, |
| JS_COMMAND_NEXT), NULL) == 0) { |
| /* idx0 has already completed - stop idx1 */ |
| kbase_gpu_stop_atom(kbdev, js, katom_idx1, |
| action); |
| ret = true; |
| } else { |
| /* idx1 is in NEXT registers - attempt to |
| * remove */ |
| kbase_reg_write(kbdev, JOB_SLOT_REG(js, |
| JS_COMMAND_NEXT), |
| JS_COMMAND_NOP, NULL); |
| |
| if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, |
| JS_HEAD_NEXT_LO), NULL) != 0 || |
| kbase_reg_read(kbdev, JOB_SLOT_REG(js, |
| JS_HEAD_NEXT_HI), NULL) != 0) { |
| /* idx1 removed successfully, will be |
| * handled in IRQ once idx0 completes */ |
| kbase_job_evicted(katom_idx1); |
| kbase_gpu_remove_atom(kbdev, katom_idx1, |
| action, |
| false); |
| } else { |
| /* idx0 has already completed - stop |
| * idx1 */ |
| kbase_gpu_stop_atom(kbdev, js, |
| katom_idx1, |
| action); |
| ret = true; |
| } |
| } |
| } |
| } |
| |
| |
| if (stop_x_dep_idx0 != -1) |
| kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0, |
| NULL, action); |
| |
| if (stop_x_dep_idx1 != -1) |
| kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1, |
| NULL, action); |
| |
| return ret; |
| } |
| |
| void kbase_gpu_cacheclean(struct kbase_device *kbdev, |
| struct kbase_jd_atom *katom) |
| { |
| /* Limit the number of loops to avoid a hang if the interrupt is missed |
| */ |
| u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; |
| unsigned long flags; |
| |
| mutex_lock(&kbdev->cacheclean_lock); |
| |
| /* use GPU_COMMAND completion solution */ |
| /* clean & invalidate the caches */ |
| KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); |
| kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), |
| GPU_COMMAND_CLEAN_INV_CACHES, NULL); |
| |
| /* wait for cache flush to complete before continuing */ |
| while (--max_loops && |
| (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & |
| CLEAN_CACHES_COMPLETED) == 0) |
| ; |
| |
| /* clear the CLEAN_CACHES_COMPLETED irq */ |
| KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, |
| CLEAN_CACHES_COMPLETED); |
| kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), |
| CLEAN_CACHES_COMPLETED, NULL); |
| KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state != |
| KBASE_INSTR_STATE_CLEANING, |
| "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang."); |
| |
| mutex_unlock(&kbdev->cacheclean_lock); |
| |
| spin_lock_irqsave(&kbdev->hwaccess_lock, flags); |
| kbase_pm_unrequest_cores(kbdev, false, |
| katom->need_cache_flush_cores_retained); |
| spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); |
| } |
| |
| void kbase_backend_complete_wq(struct kbase_device *kbdev, |
| struct kbase_jd_atom *katom) |
| { |
| /* |
| * If cache flush required due to HW workaround then perform the flush |
| * now |
| */ |
| if (katom->need_cache_flush_cores_retained) { |
| kbase_gpu_cacheclean(kbdev, katom); |
| katom->need_cache_flush_cores_retained = 0; |
| } |
| |
| if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) && |
| (katom->core_req & BASE_JD_REQ_FS) && |
| katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT && |
| (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) && |
| !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) { |
| dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n"); |
| if (kbasep_10969_workaround_clamp_coordinates(katom)) { |
| /* The job had a TILE_RANGE_FAULT after was soft-stopped |
| * Due to an HW issue we try to execute the job again. |
| */ |
| dev_dbg(kbdev->dev, |
| "Clamping has been executed, try to rerun the job\n" |
| ); |
| katom->event_code = BASE_JD_EVENT_STOPPED; |
| katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN; |
| } |
| } |
| |
| /* Clear the coreref_state now - while check_deref_cores() may not have |
| * been called yet, the caller will have taken a copy of this field. If |
| * this is not done, then if the atom is re-scheduled (following a soft |
| * stop) then the core reference would not be retaken. */ |
| katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; |
| katom->affinity = 0; |
| } |
| |
| void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, |
| base_jd_core_req core_req, u64 affinity, |
| enum kbase_atom_coreref_state coreref_state) |
| { |
| unsigned long flags; |
| |
| spin_lock_irqsave(&kbdev->hwaccess_lock, flags); |
| kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity, |
| coreref_state); |
| spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); |
| |
| if (!kbdev->pm.active_count) { |
| mutex_lock(&kbdev->js_data.runpool_mutex); |
| mutex_lock(&kbdev->pm.lock); |
| kbase_pm_update_active(kbdev); |
| mutex_unlock(&kbdev->pm.lock); |
| mutex_unlock(&kbdev->js_data.runpool_mutex); |
| } |
| } |
| |
| void kbase_gpu_dump_slots(struct kbase_device *kbdev) |
| { |
| struct kbasep_js_device_data *js_devdata; |
| unsigned long flags; |
| int js; |
| |
| js_devdata = &kbdev->js_data; |
| |
| spin_lock_irqsave(&kbdev->hwaccess_lock, flags); |
| |
| dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n"); |
| |
| for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { |
| int idx; |
| |
| for (idx = 0; idx < SLOT_RB_SIZE; idx++) { |
| struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, |
| js, |
| idx); |
| |
| if (katom) |
| dev_info(kbdev->dev, |
| " js%d idx%d : katom=%p gpu_rb_state=%d\n", |
| js, idx, katom, katom->gpu_rb_state); |
| else |
| dev_info(kbdev->dev, " js%d idx%d : empty\n", |
| js, idx); |
| } |
| } |
| |
| spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); |
| } |
| |
| |
| |