| /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ |
| /* |
| * |
| * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. |
| * |
| * This program is free software and is provided to you under the terms of the |
| * GNU General Public License version 2 as published by the Free Software |
| * Foundation, and any use by you of this program is subject to the terms |
| * of such GNU license. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, you can access it online at |
| * http://www.gnu.org/licenses/gpl-2.0.html. |
| * |
| */ |
| |
| /* |
| * Definitions (types, defines, etcs) specific to Job Manager Kbase. |
| * They are placed here to allow the hierarchy of header files to work. |
| */ |
| |
| #ifndef _KBASE_JM_DEFS_H_ |
| #define _KBASE_JM_DEFS_H_ |
| |
| #include "mali_kbase_js_defs.h" |
| |
| /* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */ |
| #define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1 |
| |
| /* |
| * Number of milliseconds before resetting the GPU when a job cannot be "zapped" |
| * from the hardware. Note that the time is actually |
| * ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and |
| * the GPU actually being reset to give other contexts time for their jobs |
| * to be soft-stopped and removed from the hardware before resetting. |
| */ |
| #define ZAP_TIMEOUT 1000 |
| |
| /* |
| * Prevent soft-stops from occurring in scheduling situations |
| * |
| * This is not due to HW issues, but when scheduling is desired to be more |
| * predictable. |
| * |
| * Therefore, soft stop may still be disabled due to HW issues. |
| * |
| * Soft stop will still be used for non-scheduling purposes e.g. when |
| * terminating a context. |
| * |
| * if not in use, define this value to 0 instead of being undefined. |
| */ |
| #define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0 |
| |
| /* |
| * Prevent hard-stops from occurring in scheduling situations |
| * |
| * This is not due to HW issues, but when scheduling is desired to be more |
| * predictable. |
| * |
| * Hard stop will still be used for non-scheduling purposes e.g. when |
| * terminating a context. |
| * |
| * if not in use, define this value to 0 instead of being undefined. |
| */ |
| #define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 |
| |
| /* Atom has been previously soft-stopped */ |
| #define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1<<1) |
| /* Atom has been previously retried to execute */ |
| #define KBASE_KATOM_FLAGS_RERUN (1<<2) |
| /* Atom submitted with JOB_CHAIN_FLAG bit set in JS_CONFIG_NEXT register, helps |
| * to disambiguate short-running job chains during soft/hard stopping of jobs |
| */ |
| #define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3) |
| /* Atom has been previously hard-stopped. */ |
| #define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4) |
| /* Atom has caused us to enter disjoint state */ |
| #define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5) |
| /* Atom blocked on cross-slot dependency */ |
| #define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7) |
| /* Atom has fail dependency on cross-slot dependency */ |
| #define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8) |
| /* Atom is currently in the list of atoms blocked on cross-slot dependencies */ |
| #define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9) |
| /* Atom is currently holding a context reference */ |
| #define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10) |
| /* Atom requires GPU to be in protected mode */ |
| #define KBASE_KATOM_FLAG_PROTECTED (1<<11) |
| /* Atom has been stored in runnable_tree */ |
| #define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12) |
| /* Atom is waiting for L2 caches to power up in order to enter protected mode */ |
| #define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1<<13) |
| |
| /* SW related flags about types of JS_COMMAND action |
| * NOTE: These must be masked off by JS_COMMAND_MASK |
| */ |
| |
| /* This command causes a disjoint event */ |
| #define JS_COMMAND_SW_CAUSES_DISJOINT 0x100 |
| |
| /* Bitmask of all SW related flags */ |
| #define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT) |
| |
| #if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK) |
| #error "JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK." \ |
| "Must update JS_COMMAND_SW_<..> bitmasks" |
| #endif |
| |
| /* Soft-stop command that causes a Disjoint event. This of course isn't |
| * entirely masked off by JS_COMMAND_MASK |
| */ |
| #define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \ |
| (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP) |
| |
| #define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT |
| |
| /* Serialize atoms within a slot (ie only one atom per job slot) */ |
| #define KBASE_SERIALIZE_INTRA_SLOT (1 << 0) |
| /* Serialize atoms between slots (ie only one job slot running at any time) */ |
| #define KBASE_SERIALIZE_INTER_SLOT (1 << 1) |
| /* Reset the GPU after each atom completion */ |
| #define KBASE_SERIALIZE_RESET (1 << 2) |
| |
| #if IS_ENABLED(CONFIG_DEBUG_FS) |
| /** |
| * struct base_job_fault_event - keeps track of the atom which faulted or which |
| * completed after the faulty atom but before the |
| * debug data for faulty atom was dumped. |
| * |
| * @event_code: event code for the atom, should != BASE_JD_EVENT_DONE for |
| * the atom which faulted. |
| * @katom: pointer to the atom for which job fault occurred or which |
| * completed after the faulty atom. |
| * @job_fault_work: work item, queued only for the faulty atom, which waits for |
| * the dumping to get completed and then does the bottom half |
| * of job done for the atoms which followed the faulty atom. |
| * @head: List head used to store the atom in the global list of |
| * faulty atoms or context specific list of atoms which got |
| * completed during the dump. |
| * @reg_offset: offset of the register to be dumped next, only applicable |
| * for the faulty atom. |
| */ |
| struct base_job_fault_event { |
| |
| u32 event_code; |
| struct kbase_jd_atom *katom; |
| struct work_struct job_fault_work; |
| struct list_head head; |
| int reg_offset; |
| }; |
| #endif |
| |
| /** |
| * struct kbase_jd_atom_dependency - Contains the dependency info for an atom. |
| * @atom: pointer to the dependee atom. |
| * @dep_type: type of dependency on the dependee @atom, i.e. order or data |
| * dependency. BASE_JD_DEP_TYPE_INVALID indicates no dependency. |
| */ |
| struct kbase_jd_atom_dependency { |
| struct kbase_jd_atom *atom; |
| u8 dep_type; |
| }; |
| |
| /** |
| * kbase_jd_katom_dep_atom - Retrieves a read-only reference to the |
| * dependee atom. |
| * @dep: pointer to the dependency info structure. |
| * |
| * Return: readonly reference to dependee atom. |
| */ |
| static inline const struct kbase_jd_atom * |
| kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep) |
| { |
| LOCAL_ASSERT(dep != NULL); |
| |
| return (const struct kbase_jd_atom *)(dep->atom); |
| } |
| |
| /** |
| * kbase_jd_katom_dep_type - Retrieves the dependency type info |
| * |
| * @dep: pointer to the dependency info structure. |
| * |
| * Return: the type of dependency there is on the dependee atom. |
| */ |
| static inline u8 kbase_jd_katom_dep_type( |
| const struct kbase_jd_atom_dependency *dep) |
| { |
| LOCAL_ASSERT(dep != NULL); |
| |
| return dep->dep_type; |
| } |
| |
| /** |
| * kbase_jd_katom_dep_set - sets up the dependency info structure |
| * as per the values passed. |
| * @const_dep: pointer to the dependency info structure to be setup. |
| * @a: pointer to the dependee atom. |
| * @type: type of dependency there is on the dependee atom. |
| */ |
| static inline void kbase_jd_katom_dep_set( |
| const struct kbase_jd_atom_dependency *const_dep, |
| struct kbase_jd_atom *a, u8 type) |
| { |
| struct kbase_jd_atom_dependency *dep; |
| |
| LOCAL_ASSERT(const_dep != NULL); |
| |
| dep = (struct kbase_jd_atom_dependency *)const_dep; |
| |
| dep->atom = a; |
| dep->dep_type = type; |
| } |
| |
| /** |
| * kbase_jd_katom_dep_clear - resets the dependency info structure |
| * |
| * @const_dep: pointer to the dependency info structure to be setup. |
| */ |
| static inline void kbase_jd_katom_dep_clear( |
| const struct kbase_jd_atom_dependency *const_dep) |
| { |
| struct kbase_jd_atom_dependency *dep; |
| |
| LOCAL_ASSERT(const_dep != NULL); |
| |
| dep = (struct kbase_jd_atom_dependency *)const_dep; |
| |
| dep->atom = NULL; |
| dep->dep_type = BASE_JD_DEP_TYPE_INVALID; |
| } |
| |
| /** |
| * enum kbase_atom_gpu_rb_state - The state of an atom, pertinent after it |
| * becomes runnable, with respect to job slot |
| * ringbuffer/fifo. |
| * @KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: Atom not currently present in slot fifo, |
| * which implies that either atom has not become |
| * runnable due to dependency or has completed |
| * the execution on GPU. |
| * @KBASE_ATOM_GPU_RB_WAITING_BLOCKED: Atom has been added to slot fifo but is |
| * blocked due to cross slot dependency, |
| * can't be submitted to GPU. |
| * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: Atom has been added to slot |
| * fifo but is waiting for the completion of |
| * previously added atoms in current & other |
| * slots, as their protected mode requirements |
| * do not match with the current atom. |
| * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: Atom is in slot fifo |
| * and is waiting for completion of protected |
| * mode transition, needed before the atom is |
| * submitted to GPU. |
| * @KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: Atom is in slot fifo but is |
| * waiting for the cores, which are needed to |
| * execute the job chain represented by the atom, |
| * to become available |
| * @KBASE_ATOM_GPU_RB_READY: Atom is in slot fifo and can be submitted to |
| * GPU. |
| * @KBASE_ATOM_GPU_RB_SUBMITTED: Atom is in slot fifo and has been submitted |
| * to GPU. |
| * @KBASE_ATOM_GPU_RB_RETURN_TO_JS: Atom must be returned to JS due to some |
| * failure, but only after the previously added |
| * atoms in fifo have completed or have also |
| * been returned to JS. |
| */ |
| enum kbase_atom_gpu_rb_state { |
| KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB, |
| KBASE_ATOM_GPU_RB_WAITING_BLOCKED, |
| KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV, |
| KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION, |
| KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE, |
| KBASE_ATOM_GPU_RB_READY, |
| KBASE_ATOM_GPU_RB_SUBMITTED, |
| KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1 |
| }; |
| |
| /** |
| * enum kbase_atom_enter_protected_state - The state of an atom with respect to |
| * the preparation for GPU's entry into protected mode, |
| * becomes pertinent only after atom's state with respect |
| * to slot ringbuffer is |
| * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION |
| * @KBASE_ATOM_ENTER_PROTECTED_CHECK: Starting state. Check if there are any |
| * atoms currently submitted to GPU and protected mode |
| * transition is not already in progress. |
| * @KBASE_ATOM_ENTER_PROTECTED_HWCNT: Wait for hardware counter context to |
| * become disabled before entry into protected mode. |
| * @KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: Wait for the L2 to become idle in |
| * preparation for the coherency change. L2 shall be |
| * powered down and GPU shall come out of fully |
| * coherent mode before entering protected mode. |
| * @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change; |
| * for BASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on |
| * so that coherency register contains correct value when |
| * GPU enters protected mode. |
| * @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for |
| * BASE_HW_ISSUE_TGOX_R1_1234 check |
| * that L2 is powered up and switch GPU to protected mode. |
| */ |
| enum kbase_atom_enter_protected_state { |
| /* |
| * NOTE: The integer value of this must match |
| * KBASE_ATOM_EXIT_PROTECTED_CHECK. |
| */ |
| KBASE_ATOM_ENTER_PROTECTED_CHECK = 0, |
| KBASE_ATOM_ENTER_PROTECTED_HWCNT, |
| KBASE_ATOM_ENTER_PROTECTED_IDLE_L2, |
| KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY, |
| KBASE_ATOM_ENTER_PROTECTED_FINISHED, |
| }; |
| |
| /** |
| * enum kbase_atom_exit_protected_state - The state of an atom with respect to |
| * the preparation for GPU's exit from protected mode, |
| * becomes pertinent only after atom's state with respect |
| * to slot ngbuffer is |
| * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION |
| * @KBASE_ATOM_EXIT_PROTECTED_CHECK: Starting state. Check if there are any |
| * atoms currently submitted to GPU and protected mode |
| * transition is not already in progress. |
| * @KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: Wait for the L2 to become idle in |
| * preparation for the reset, as exiting protected mode |
| * requires a reset. |
| * @KBASE_ATOM_EXIT_PROTECTED_RESET: Issue the reset to trigger exit from |
| * protected mode |
| * @KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: End state, Wait for the reset to |
| * complete |
| */ |
| enum kbase_atom_exit_protected_state { |
| /* |
| * NOTE: The integer value of this must match |
| * KBASE_ATOM_ENTER_PROTECTED_CHECK. |
| */ |
| KBASE_ATOM_EXIT_PROTECTED_CHECK = 0, |
| KBASE_ATOM_EXIT_PROTECTED_IDLE_L2, |
| KBASE_ATOM_EXIT_PROTECTED_RESET, |
| KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT, |
| }; |
| |
| /** |
| * struct kbase_ext_res - Contains the info for external resources referred |
| * by an atom, which have been mapped on GPU side. |
| * @gpu_address: Start address of the memory region allocated for |
| * the resource from GPU virtual address space. |
| * @alloc: pointer to physical pages tracking object, set on |
| * mapping the external resource on GPU side. |
| */ |
| struct kbase_ext_res { |
| u64 gpu_address; |
| struct kbase_mem_phy_alloc *alloc; |
| }; |
| |
| /** |
| * struct kbase_jd_atom - object representing the atom, containing the complete |
| * state and attributes of an atom. |
| * @work: work item for the bottom half processing of the atom, |
| * by JD or JS, after it got executed on GPU or the |
| * input fence got signaled |
| * @start_timestamp: time at which the atom was submitted to the GPU, by |
| * updating the JS_HEAD_NEXTn register. |
| * @udata: copy of the user data sent for the atom in |
| * base_jd_submit. |
| * @kctx: Pointer to the base context with which the atom is |
| * associated. |
| * @dep_head: Array of 2 list heads, pointing to the two list of |
| * atoms |
| * which are blocked due to dependency on this atom. |
| * @dep_item: Array of 2 list heads, used to store the atom in the |
| * list of other atoms depending on the same dependee |
| * atom. |
| * @dep: Array containing the dependency info for the 2 atoms |
| * on which the atom depends upon. |
| * @jd_item: List head used during job dispatch job_done |
| * processing - as dependencies may not be entirely |
| * resolved at this point, |
| * we need to use a separate list head. |
| * @in_jd_list: flag set to true if atom's @jd_item is currently on |
| * a list, prevents atom being processed twice. |
| * @jit_ids: Zero-terminated array of IDs of just-in-time memory |
| * allocations written to by the atom. When the atom |
| * completes, the value stored at the |
| * &struct_base_jit_alloc_info.heap_info_gpu_addr of |
| * each allocation is read in order to enforce an |
| * overall physical memory usage limit. |
| * @nr_extres: number of external resources referenced by the atom. |
| * @extres: pointer to the location containing info about |
| * @nr_extres external resources referenced by the atom. |
| * @device_nr: indicates the coregroup with which the atom is |
| * associated, when |
| * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified. |
| * @jc: GPU address of the job-chain. |
| * @softjob_data: Copy of data read from the user space buffer that @jc |
| * points to. |
| * @fence: Stores either an input or output sync fence, |
| * depending on soft-job type |
| * @sync_waiter: Pointer to the sync fence waiter structure passed to |
| * the callback function on signaling of the input |
| * fence. |
| * @dma_fence: object containing pointers to both input & output |
| * fences and other related members used for explicit |
| * sync through soft jobs and for the implicit |
| * synchronization required on access to external |
| * resources. |
| * @dma_fence.fence_in: Input fence |
| * @dma_fence.fence: Points to the dma-buf output fence for this atom. |
| * @dma_fence.context: The dma-buf fence context number for this atom. A |
| * unique context number is allocated to each katom in |
| * the context on context creation. |
| * @dma_fence.seqno: The dma-buf fence sequence number for this atom. This |
| * is increased every time this katom uses dma-buf fence |
| * @dma_fence.callbacks: List of all callbacks set up to wait on other fences |
| * @dma_fence.dep_count: Atomic counter of number of outstandind dma-buf fence |
| * dependencies for this atom. |
| * @event_code: Event code for the job chain represented by the atom, |
| * both HW and low-level SW events are represented by |
| * event codes. |
| * @core_req: bitmask of BASE_JD_REQ_* flags specifying either |
| * Hw or Sw requirements for the job chain represented |
| * by the atom. |
| * @ticks: Number of scheduling ticks for which atom has been |
| * running on the GPU. |
| * @sched_priority: Priority of the atom for Job scheduling, as per the |
| * KBASE_JS_ATOM_SCHED_PRIO_*. |
| * @completed: Wait queue to wait upon for the completion of atom. |
| * @status: Indicates at high level at what stage the atom is in, |
| * as per KBASE_JD_ATOM_STATE_*, that whether it is not |
| * in use or its queued in JD or given to JS or |
| * submitted to Hw or it completed the execution on Hw. |
| * @work_id: used for GPU tracepoints, its a snapshot of the |
| * 'work_id' counter in kbase_jd_context which is |
| * incremented on every call to base_jd_submit. |
| * @slot_nr: Job slot chosen for the atom. |
| * @atom_flags: bitmask of KBASE_KATOM_FLAG* flags capturing the |
| * excat low level state of the atom. |
| * @gpu_rb_state: bitmnask of KBASE_ATOM_GPU_RB_* flags, precisely |
| * tracking atom's state after it has entered |
| * Job scheduler on becoming runnable. Atom |
| * could be blocked due to cross slot dependency |
| * or waiting for the shader cores to become available |
| * or waiting for protected mode transitions to |
| * complete. |
| * @need_cache_flush_cores_retained: flag indicating that manual flush of GPU |
| * cache is needed for the atom and the shader cores |
| * used for atom have been kept on. |
| * @blocked: flag indicating that atom's resubmission to GPU is |
| * blocked till the work item is scheduled to return the |
| * atom to JS. |
| * @seq_nr: user-space sequence number, to order atoms in some |
| * temporal order |
| * @pre_dep: Pointer to atom that this atom has same-slot |
| * dependency on |
| * @post_dep: Pointer to atom that has same-slot dependency on |
| * this atom |
| * @x_pre_dep: Pointer to atom that this atom has cross-slot |
| * dependency on |
| * @x_post_dep: Pointer to atom that has cross-slot dependency on |
| * this atom |
| * @flush_id: The GPU's flush count recorded at the time of |
| * submission, |
| * used for the cache flush optimization |
| * @fault_event: Info for dumping the debug data on Job fault. |
| * @queue: List head used for 4 different purposes : |
| * Adds atom to the list of dma-buf fence waiting atoms. |
| * Adds atom to the list of atoms blocked due to cross |
| * slot dependency. |
| * Adds atom to the list of softjob atoms for which JIT |
| * allocation has been deferred |
| * Adds atom to the list of softjob atoms waiting for |
| * the signaling of fence. |
| * @jit_node: Used to keep track of all JIT free/alloc jobs in |
| * submission order |
| * @jit_blocked: Flag indicating that JIT allocation requested through |
| * softjob atom will be reattempted after the impending |
| * free of other active JIT allocations. |
| * @will_fail_event_code: If non-zero, this indicates that the atom will fail |
| * with the set event_code when the atom is processed. |
| * Used for special handling of atoms, which have a data |
| * dependency on the failed atoms. |
| * @protected_state: State of the atom, as per |
| * KBASE_ATOM_(ENTER|EXIT)_PROTECTED_*, |
| * when transitioning into or out of protected mode. |
| * Atom will be either entering or exiting the |
| * protected mode. |
| * @protected_state.enter: entering the protected mode. |
| * @protected_state.exit: exiting the protected mode. |
| * @runnable_tree_node: The node added to context's job slot specific rb tree |
| * when the atom becomes runnable. |
| * @age: Age of atom relative to other atoms in the context, |
| * is snapshot of the age_count counter in kbase |
| * context. |
| * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. |
| * @renderpass_id:Renderpass identifier used to associate an atom that has |
| * BASE_JD_REQ_START_RENDERPASS set in its core requirements |
| * with an atom that has BASE_JD_REQ_END_RENDERPASS set. |
| * @jc_fragment: Set of GPU fragment job chains |
| * @retry_count: TODO: Not used,to be removed |
| */ |
| struct kbase_jd_atom { |
| struct work_struct work; |
| ktime_t start_timestamp; |
| |
| struct base_jd_udata udata; |
| struct kbase_context *kctx; |
| |
| struct list_head dep_head[2]; |
| struct list_head dep_item[2]; |
| const struct kbase_jd_atom_dependency dep[2]; |
| struct list_head jd_item; |
| bool in_jd_list; |
| |
| #if MALI_JIT_PRESSURE_LIMIT_BASE |
| u8 jit_ids[2]; |
| #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ |
| |
| u16 nr_extres; |
| struct kbase_ext_res *extres; |
| |
| u32 device_nr; |
| u64 jc; |
| void *softjob_data; |
| #if defined(CONFIG_SYNC) |
| struct sync_fence *fence; |
| struct sync_fence_waiter sync_waiter; |
| #endif /* CONFIG_SYNC */ |
| #if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) |
| struct { |
| /* Use the functions/API defined in mali_kbase_fence.h to |
| * when working with this sub struct |
| */ |
| #if defined(CONFIG_SYNC_FILE) |
| #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) |
| struct fence *fence_in; |
| #else |
| struct dma_fence *fence_in; |
| #endif |
| #endif |
| /* This points to the dma-buf output fence for this atom. If |
| * this is NULL then there is no fence for this atom and the |
| * following fields related to dma_fence may have invalid data. |
| * |
| * The context and seqno fields contain the details for this |
| * fence. |
| * |
| * This fence is signaled when the katom is completed, |
| * regardless of the event_code of the katom (signal also on |
| * failure). |
| */ |
| #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) |
| struct fence *fence; |
| #else |
| struct dma_fence *fence; |
| #endif |
| unsigned int context; |
| atomic_t seqno; |
| /* This contains a list of all callbacks set up to wait on |
| * other fences. This atom must be held back from JS until all |
| * these callbacks have been called and dep_count have reached |
| * 0. The initial value of dep_count must be equal to the |
| * number of callbacks on this list. |
| * |
| * This list is protected by jctx.lock. Callbacks are added to |
| * this list when the atom is built and the wait are set up. |
| * All the callbacks then stay on the list until all callbacks |
| * have been called and the atom is queued, or cancelled, and |
| * then all callbacks are taken off the list and freed. |
| */ |
| struct list_head callbacks; |
| /* Atomic counter of number of outstandind dma-buf fence |
| * dependencies for this atom. When dep_count reaches 0 the |
| * atom may be queued. |
| * |
| * The special value "-1" may only be set after the count |
| * reaches 0, while holding jctx.lock. This indicates that the |
| * atom has been handled, either queued in JS or cancelled. |
| * |
| * If anyone but the dma-fence worker sets this to -1 they must |
| * ensure that any potentially queued worker must have |
| * completed before allowing the atom to be marked as unused. |
| * This can be done by flushing the fence work queue: |
| * kctx->dma_fence.wq. |
| */ |
| atomic_t dep_count; |
| } dma_fence; |
| #endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE */ |
| |
| /* Note: refer to kbasep_js_atom_retained_state, which will take a copy |
| * of some of the following members |
| */ |
| enum base_jd_event_code event_code; |
| base_jd_core_req core_req; |
| u8 jobslot; |
| u8 renderpass_id; |
| struct base_jd_fragment jc_fragment; |
| |
| u32 ticks; |
| int sched_priority; |
| |
| wait_queue_head_t completed; |
| enum kbase_jd_atom_state status; |
| #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) |
| int work_id; |
| #endif |
| int slot_nr; |
| |
| u32 atom_flags; |
| |
| int retry_count; |
| |
| enum kbase_atom_gpu_rb_state gpu_rb_state; |
| |
| bool need_cache_flush_cores_retained; |
| |
| atomic_t blocked; |
| |
| u64 seq_nr; |
| |
| struct kbase_jd_atom *pre_dep; |
| struct kbase_jd_atom *post_dep; |
| |
| struct kbase_jd_atom *x_pre_dep; |
| struct kbase_jd_atom *x_post_dep; |
| |
| u32 flush_id; |
| |
| #if IS_ENABLED(CONFIG_DEBUG_FS) |
| struct base_job_fault_event fault_event; |
| #endif |
| struct list_head queue; |
| |
| struct list_head jit_node; |
| bool jit_blocked; |
| |
| enum base_jd_event_code will_fail_event_code; |
| |
| union { |
| enum kbase_atom_enter_protected_state enter; |
| enum kbase_atom_exit_protected_state exit; |
| } protected_state; |
| |
| struct rb_node runnable_tree_node; |
| |
| u32 age; |
| }; |
| |
| static inline bool kbase_jd_katom_is_protected( |
| const struct kbase_jd_atom *katom) |
| { |
| return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED); |
| } |
| |
| /* |
| * Theory of operations: |
| * |
| * Atom objects are statically allocated within the context structure. |
| * |
| * Each atom is the head of two lists, one for the "left" set of dependencies, |
| * one for the "right" set. |
| */ |
| |
| #define KBASE_JD_DEP_QUEUE_SIZE 256 |
| |
| /** |
| * enum kbase_jd_renderpass_state - State of a renderpass |
| * @KBASE_JD_RP_COMPLETE: Unused or completed renderpass. Can only transition to |
| * START. |
| * @KBASE_JD_RP_START: Renderpass making a first attempt at tiling. |
| * Can transition to PEND_OOM or COMPLETE. |
| * @KBASE_JD_RP_PEND_OOM: Renderpass whose first attempt at tiling used too much |
| * memory and has a soft-stop pending. Can transition to |
| * OOM or COMPLETE. |
| * @KBASE_JD_RP_OOM: Renderpass whose first attempt at tiling used too much |
| * memory and therefore switched to incremental |
| * rendering. The fragment job chain is forced to run. |
| * Can only transition to RETRY. |
| * @KBASE_JD_RP_RETRY: Renderpass making a second or subsequent attempt at |
| * tiling. Can transition to RETRY_PEND_OOM or COMPLETE. |
| * @KBASE_JD_RP_RETRY_PEND_OOM: Renderpass whose second or subsequent attempt at |
| * tiling used too much memory again and has a |
| * soft-stop pending. Can transition to RETRY_OOM |
| * or COMPLETE. |
| * @KBASE_JD_RP_RETRY_OOM: Renderpass whose second or subsequent attempt at |
| * tiling used too much memory again. The fragment job |
| * chain is forced to run. Can only transition to RETRY. |
| * |
| * A state machine is used to control incremental rendering. |
| */ |
| enum kbase_jd_renderpass_state { |
| KBASE_JD_RP_COMPLETE, /* COMPLETE => START */ |
| KBASE_JD_RP_START, /* START => PEND_OOM or COMPLETE */ |
| KBASE_JD_RP_PEND_OOM, /* PEND_OOM => OOM or COMPLETE */ |
| KBASE_JD_RP_OOM, /* OOM => RETRY */ |
| KBASE_JD_RP_RETRY, /* RETRY => RETRY_PEND_OOM or |
| * COMPLETE |
| */ |
| KBASE_JD_RP_RETRY_PEND_OOM, /* RETRY_PEND_OOM => RETRY_OOM or |
| * COMPLETE |
| */ |
| KBASE_JD_RP_RETRY_OOM, /* RETRY_OOM => RETRY */ |
| }; |
| |
| /** |
| * struct kbase_jd_renderpass - Data for a renderpass |
| * @state: Current state of the renderpass. If KBASE_JD_RP_COMPLETE then |
| * all other members are invalid. |
| * Both the job dispatcher context and hwaccess_lock must be |
| * locked to modify this so that it can be read with either |
| * (or both) locked. |
| * @start_katom: Address of the atom that is the start of a renderpass. |
| * Both the job dispatcher context and hwaccess_lock must be |
| * locked to modify this so that it can be read with either |
| * (or both) locked. |
| * @end_katom: Address of the atom that is the end of a renderpass, or NULL |
| * if that atom hasn't been added to the job scheduler yet. |
| * The job dispatcher context and hwaccess_lock must be |
| * locked to modify this so that it can be read with either |
| * (or both) locked. |
| * @oom_reg_list: A list of region structures which triggered out-of-memory. |
| * The hwaccess_lock must be locked to access this. |
| * |
| * Atoms tagged with BASE_JD_REQ_START_RENDERPASS or BASE_JD_REQ_END_RENDERPASS |
| * are associated with an object of this type, which is created and maintained |
| * by kbase to keep track of each renderpass. |
| */ |
| struct kbase_jd_renderpass { |
| enum kbase_jd_renderpass_state state; |
| struct kbase_jd_atom *start_katom; |
| struct kbase_jd_atom *end_katom; |
| struct list_head oom_reg_list; |
| }; |
| |
| /** |
| * struct kbase_jd_context - per context object encapsulating all the |
| * Job dispatcher related state. |
| * @lock: lock to serialize the updates made to the |
| * Job dispatcher state and kbase_jd_atom objects. |
| * @sched_info: Structure encapsulating all the Job scheduling |
| * info. |
| * @atoms: Array of the objects representing atoms, |
| * containing the complete state and attributes |
| * of an atom. |
| * @renderpasses: Array of renderpass state for incremental |
| * rendering, indexed by user-specified renderpass |
| * ID. |
| * @job_nr: Tracks the number of atoms being processed by the |
| * kbase. This includes atoms that are not tracked by |
| * scheduler: 'not ready to run' & 'dependency-only' |
| * jobs. |
| * @zero_jobs_wait: Waitq that reflects whether there are no jobs |
| * (including SW-only dependency jobs). This is set |
| * when no jobs are present on the ctx, and clear |
| * when there are jobs. |
| * This must be updated atomically with @job_nr. |
| * note: Job Dispatcher knows about more jobs than |
| * the Job Scheduler as it is unaware of jobs that |
| * are blocked on dependencies and SW-only dependency |
| * jobs. This waitq can be waited upon to find out |
| * when the context jobs are all done/cancelled |
| * (including those that might've been blocked |
| * on dependencies) - and so, whether it can be |
| * terminated. However, it should only be terminated |
| * once it is not present in the run-pool. |
| * Since the waitq is only set under @lock, |
| * the waiter should also briefly obtain and drop |
| * @lock to guarantee that the setter has completed |
| * its work on the kbase_context |
| * @job_done_wq: Workqueue to which the per atom work item is |
| * queued for bottom half processing when the |
| * atom completes |
| * execution on GPU or the input fence get signaled. |
| * @tb_lock: Lock to serialize the write access made to @tb to |
| * to store the register access trace messages. |
| * @tb: Pointer to the Userspace accessible buffer storing |
| * the trace messages for register read/write |
| * accesses made by the Kbase. The buffer is filled |
| * in circular fashion. |
| * @tb_wrap_offset: Offset to the end location in the trace buffer, |
| * the write pointer is moved to the beginning on |
| * reaching this offset. |
| * @work_id: atomic variable used for GPU tracepoints, |
| * incremented on every call to base_jd_submit. |
| * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both |
| * allocate & free, in submission order, protected |
| * by kbase_jd_context.lock. |
| * @jit_pending_alloc: A list of just-in-time memory allocation |
| * soft-jobs which will be reattempted after the |
| * impending free of other active allocations. |
| * @max_priority: Max priority level allowed for this context. |
| */ |
| struct kbase_jd_context { |
| struct mutex lock; |
| struct kbasep_js_kctx_info sched_info; |
| struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; |
| struct kbase_jd_renderpass renderpasses[BASE_JD_RP_COUNT]; |
| struct workqueue_struct *job_done_wq; |
| |
| wait_queue_head_t zero_jobs_wait; |
| spinlock_t tb_lock; |
| u32 *tb; |
| u32 job_nr; |
| size_t tb_wrap_offset; |
| |
| #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) |
| atomic_t work_id; |
| #endif |
| |
| struct list_head jit_atoms_head; |
| struct list_head jit_pending_alloc; |
| int max_priority; |
| }; |
| |
| /** |
| * struct jsctx_queue - JS context atom queue |
| * @runnable_tree: Root of RB-tree containing currently runnable atoms on this |
| * job slot. |
| * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot |
| * dependencies. Atoms on this list will be moved to the |
| * runnable_tree when the blocking atom completes. |
| * |
| * hwaccess_lock must be held when accessing this structure. |
| */ |
| struct jsctx_queue { |
| struct rb_root runnable_tree; |
| struct list_head x_dep_head; |
| }; |
| |
| /** |
| * struct kbase_as - Object representing an address space of GPU. |
| * @number: Index at which this address space structure is present |
| * in an array of address space structures embedded inside |
| * the &struct kbase_device. |
| * @pf_wq: Workqueue for processing work items related to |
| * Page fault and Bus fault handling. |
| * @work_pagefault: Work item for the Page fault handling. |
| * @work_busfault: Work item for the Bus fault handling. |
| * @pf_data: Data relating to Page fault. |
| * @bf_data: Data relating to Bus fault. |
| * @current_setup: Stores the MMU configuration for this address space. |
| */ |
| struct kbase_as { |
| int number; |
| struct workqueue_struct *pf_wq; |
| struct work_struct work_pagefault; |
| struct work_struct work_busfault; |
| struct kbase_fault pf_data; |
| struct kbase_fault bf_data; |
| struct kbase_mmu_setup current_setup; |
| }; |
| |
| #endif /* _KBASE_JM_DEFS_H_ */ |
