| /**************************************************************************** |
| * |
| * The MIT License (MIT) |
| * |
| * Copyright (c) 2014 - 2016 Vivante Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be included in |
| * all copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| * DEALINGS IN THE SOFTWARE. |
| * |
| ***************************************************************************** |
| * |
| * The GPL License (GPL) |
| * |
| * Copyright (C) 2014 - 2016 Vivante Corporation |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License |
| * as published by the Free Software Foundation; either version 2 |
| * of the License, or (at your option) any later version. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| * |
| ***************************************************************************** |
| * |
| * Note: This software is released under dual MIT and GPL licenses. A |
| * recipient may use this file under the terms of either the MIT license or |
| * GPL License. If you wish to use only one license not the other, you can |
| * indicate your decision by deleting one of the above license notices in your |
| * version of this file. |
| * |
| *****************************************************************************/ |
| |
| |
| #include "gc_hal_kernel_precomp.h" |
| #include "gc_hal_kernel_context.h" |
| |
| #define _GC_OBJ_ZONE gcvZONE_COMMAND |
| |
| /******************************************************************************\ |
| ********************************* Support Code ********************************* |
| \******************************************************************************/ |
| |
| /******************************************************************************* |
| ** |
| ** _NewQueue |
| ** |
| ** Allocate a new command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckCOMMAND Command |
| ** gckCOMMAND object has been updated with a new command queue. |
| */ |
| static gceSTATUS |
| _NewQueue( |
| IN OUT gckCOMMAND Command |
| ) |
| { |
| gceSTATUS status; |
| gctINT currentIndex, newIndex; |
| gctPHYS_ADDR_T physical; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Switch to the next command buffer. */ |
| currentIndex = Command->index; |
| newIndex = (currentIndex + 1) % gcdCOMMAND_QUEUES; |
| |
| /* Wait for availability. */ |
| #if gcdDUMP_COMMAND |
| gcmkPRINT("@[kernel.waitsignal]"); |
| #endif |
| |
| gcmkONERROR(gckOS_WaitSignal( |
| Command->os, |
| Command->queues[newIndex].signal, |
| gcvINFINITE |
| )); |
| |
| #if gcmIS_DEBUG(gcdDEBUG_TRACE) |
| if (newIndex < currentIndex) |
| { |
| Command->wrapCount += 1; |
| |
| gcmkTRACE_ZONE_N( |
| gcvLEVEL_INFO, gcvZONE_COMMAND, |
| 2 * 4, |
| "%s(%d): queue array wrapped around.\n", |
| __FUNCTION__, __LINE__ |
| ); |
| } |
| |
| gcmkTRACE_ZONE_N( |
| gcvLEVEL_INFO, gcvZONE_COMMAND, |
| 3 * 4, |
| "%s(%d): total queue wrap arounds %d.\n", |
| __FUNCTION__, __LINE__, Command->wrapCount |
| ); |
| |
| gcmkTRACE_ZONE_N( |
| gcvLEVEL_INFO, gcvZONE_COMMAND, |
| 3 * 4, |
| "%s(%d): switched to queue %d.\n", |
| __FUNCTION__, __LINE__, newIndex |
| ); |
| #endif |
| |
| /* Update gckCOMMAND object with new command queue. */ |
| Command->index = newIndex; |
| Command->newQueue = gcvTRUE; |
| Command->logical = Command->queues[newIndex].logical; |
| Command->address = Command->queues[newIndex].address; |
| Command->offset = 0; |
| |
| gcmkONERROR(gckOS_GetPhysicalAddress( |
| Command->os, |
| Command->logical, |
| &physical |
| )); |
| |
| gcmkSAFECASTPHYSADDRT(Command->physical, physical); |
| |
| if (currentIndex != -1) |
| { |
| /* Mark the command queue as available. */ |
| gcmkONERROR(gckEVENT_Signal( |
| Command->kernel->eventObj, |
| Command->queues[currentIndex].signal, |
| gcvKERNEL_COMMAND |
| )); |
| } |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("Command->index=%d", Command->index); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _IncrementCommitAtom( |
| IN gckCOMMAND Command, |
| IN gctBOOL Increment |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| gctINT32 atomValue; |
| gctBOOL powerAcquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Extract the gckHARDWARE and gckEVENT objects. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| /* Grab the power mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, hardware->powerMutex, gcvINFINITE |
| )); |
| powerAcquired = gcvTRUE; |
| |
| /* Increment the commit atom. */ |
| if (Increment) |
| { |
| gcmkONERROR(gckOS_AtomIncrement( |
| Command->os, Command->atomCommit, &atomValue |
| )); |
| } |
| else |
| { |
| gcmkONERROR(gckOS_AtomDecrement( |
| Command->os, Command->atomCommit, &atomValue |
| )); |
| } |
| |
| /* Release the power mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex( |
| Command->os, hardware->powerMutex |
| )); |
| powerAcquired = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (powerAcquired) |
| { |
| /* Release the power mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex( |
| Command->os, hardware->powerMutex |
| )); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| #if gcdSECURE_USER |
| static gceSTATUS |
| _ProcessHints( |
| IN gckCOMMAND Command, |
| IN gctUINT32 ProcessID, |
| IN gcoCMDBUF CommandBuffer |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gckKERNEL kernel; |
| gctBOOL needCopy = gcvFALSE; |
| gcskSECURE_CACHE_PTR cache; |
| gctUINT8_PTR commandBufferLogical; |
| gctUINT8_PTR hintedData; |
| gctUINT32_PTR hintArray; |
| gctUINT i, hintCount; |
| |
| gcmkHEADER_ARG( |
| "Command=0x%08X ProcessID=%d CommandBuffer=0x%08X", |
| Command, ProcessID, CommandBuffer |
| ); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Reset state array pointer. */ |
| hintArray = gcvNULL; |
| |
| /* Get the kernel object. */ |
| kernel = Command->kernel; |
| |
| /* Get the cache form the database. */ |
| gcmkONERROR(gckKERNEL_GetProcessDBCache(kernel, ProcessID, &cache)); |
| |
| /* Determine the start of the command buffer. */ |
| commandBufferLogical |
| = (gctUINT8_PTR) CommandBuffer->logical |
| + CommandBuffer->startOffset; |
| |
| /* Determine the number of records in the state array. */ |
| hintCount = CommandBuffer->hintArrayTail - CommandBuffer->hintArray; |
| |
| /* Check wehther we need to copy the structures or not. */ |
| gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy)); |
| |
| /* Get access to the state array. */ |
| if (needCopy) |
| { |
| gctUINT copySize; |
| |
| if (Command->hintArrayAllocated && |
| (Command->hintArraySize < CommandBuffer->hintArraySize)) |
| { |
| gcmkONERROR(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray))); |
| Command->hintArraySize = gcvFALSE; |
| } |
| |
| if (!Command->hintArrayAllocated) |
| { |
| gctPOINTER pointer = gcvNULL; |
| |
| gcmkONERROR(gckOS_Allocate( |
| Command->os, |
| CommandBuffer->hintArraySize, |
| &pointer |
| )); |
| |
| Command->hintArray = gcmPTR_TO_UINT64(pointer); |
| Command->hintArrayAllocated = gcvTRUE; |
| Command->hintArraySize = CommandBuffer->hintArraySize; |
| } |
| |
| hintArray = gcmUINT64_TO_PTR(Command->hintArray); |
| copySize = hintCount * gcmSIZEOF(gctUINT32); |
| |
| gcmkONERROR(gckOS_CopyFromUserData( |
| Command->os, |
| hintArray, |
| gcmUINT64_TO_PTR(CommandBuffer->hintArray), |
| copySize |
| )); |
| } |
| else |
| { |
| gctPOINTER pointer = gcvNULL; |
| |
| gcmkONERROR(gckOS_MapUserPointer( |
| Command->os, |
| gcmUINT64_TO_PTR(CommandBuffer->hintArray), |
| CommandBuffer->hintArraySize, |
| &pointer |
| )); |
| |
| hintArray = pointer; |
| } |
| |
| /* Scan through the buffer. */ |
| for (i = 0; i < hintCount; i += 1) |
| { |
| /* Determine the location of the hinted data. */ |
| hintedData = commandBufferLogical + hintArray[i]; |
| |
| /* Map handle into physical address. */ |
| gcmkONERROR(gckKERNEL_MapLogicalToPhysical( |
| kernel, cache, (gctPOINTER) hintedData |
| )); |
| } |
| |
| OnError: |
| /* Get access to the state array. */ |
| if (!needCopy && (hintArray != gcvNULL)) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| gcmUINT64_TO_PTR(CommandBuffer->hintArray), |
| CommandBuffer->hintArraySize, |
| hintArray |
| )); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| #endif |
| |
| static gceSTATUS |
| _FlushMMU( |
| IN gckCOMMAND Command |
| ) |
| { |
| #if gcdSECURITY |
| return gcvSTATUS_OK; |
| #else |
| gceSTATUS status; |
| gctUINT32 oldValue; |
| gckHARDWARE hardware = Command->kernel->hardware; |
| gctBOOL pause = gcvFALSE; |
| |
| gctUINT8_PTR pointer; |
| gctUINT32 eventBytes; |
| gctUINT32 endBytes; |
| gctUINT32 bufferSize; |
| gctUINT32 executeBytes; |
| gctUINT32 waitLinkBytes; |
| |
| gcmkONERROR(gckOS_AtomicExchange(Command->os, |
| hardware->pageTableDirty, |
| 0, |
| &oldValue)); |
| |
| if (oldValue) |
| { |
| /* Page Table is upated, flush mmu before commit. */ |
| gcmkONERROR(gckHARDWARE_FlushMMU(hardware)); |
| |
| if ((oldValue & gcvPAGE_TABLE_DIRTY_BIT_FE) |
| && (hardware->endAfterFlushMmuCache) |
| ) |
| { |
| pause = gcvTRUE; |
| } |
| } |
| |
| if (pause) |
| { |
| /* Query size. */ |
| gcmkONERROR(gckHARDWARE_Event(hardware, gcvNULL, 0, gcvKERNEL_PIXEL, &eventBytes)); |
| gcmkONERROR(gckHARDWARE_End(hardware, gcvNULL, &endBytes)); |
| |
| executeBytes = eventBytes + endBytes; |
| |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| hardware, |
| gcvNULL, |
| Command->offset + executeBytes, |
| &waitLinkBytes, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| /* Reserve space. */ |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| executeBytes, |
| (gctPOINTER *)&pointer, |
| &bufferSize |
| )); |
| |
| /* Append EVENT(29). */ |
| gcmkONERROR(gckHARDWARE_Event( |
| hardware, |
| pointer, |
| 29, |
| gcvKERNEL_PIXEL, |
| &eventBytes |
| )); |
| |
| /* Append END. */ |
| pointer += eventBytes; |
| gcmkONERROR(gckHARDWARE_End(hardware, pointer, &endBytes)); |
| |
| /* Store address to queue. */ |
| gcmkONERROR(gckENTRYQUEUE_Enqueue( |
| Command->kernel, |
| &Command->queue, |
| Command->address + Command->offset + executeBytes, |
| waitLinkBytes |
| )); |
| |
| gcmkONERROR(gckCOMMAND_Execute(Command, executeBytes)); |
| } |
| |
| return gcvSTATUS_OK; |
| OnError: |
| return status; |
| #endif |
| } |
| |
| static gceSTATUS |
| _DummyDraw( |
| IN gckCOMMAND Command |
| ) |
| { |
| #if gcdSECURITY |
| return gcvSTATUS_OK; |
| #else |
| gceSTATUS status; |
| gckHARDWARE hardware = Command->kernel->hardware; |
| |
| gctUINT8_PTR pointer; |
| gctUINT32 bufferSize; |
| |
| gctUINT32 dummyDrawBytes; |
| |
| gckHARDWARE_DummyDraw(hardware, gcvNULL, Command->queues[0].address, &dummyDrawBytes); |
| |
| /* Reserve space. */ |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| dummyDrawBytes, |
| (gctPOINTER *)&pointer, |
| &bufferSize |
| )); |
| |
| gckHARDWARE_DummyDraw(hardware, pointer, Command->queues[0].address, &dummyDrawBytes); |
| |
| gcmkONERROR(gckCOMMAND_Execute(Command, dummyDrawBytes)); |
| |
| return gcvSTATUS_OK; |
| OnError: |
| return status; |
| #endif |
| } |
| |
| static void |
| _DumpBuffer( |
| IN gctPOINTER Buffer, |
| IN gctUINT32 GpuAddress, |
| IN gctSIZE_T Size |
| ) |
| { |
| gctSIZE_T i, line, left; |
| gctUINT32_PTR data = Buffer; |
| |
| line = Size / 32; |
| left = Size % 32; |
| |
| for (i = 0; i < line; i++) |
| { |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]); |
| data += 8; |
| GpuAddress += 8 * 4; |
| } |
| |
| switch(left) |
| { |
| case 28: |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6]); |
| break; |
| case 24: |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5]); |
| break; |
| case 20: |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4]); |
| break; |
| case 16: |
| gcmkPRINT("%08X : %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3]); |
| break; |
| case 12: |
| gcmkPRINT("%08X : %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2]); |
| break; |
| case 8: |
| gcmkPRINT("%08X : %08X %08X", |
| GpuAddress, data[0], data[1]); |
| break; |
| case 4: |
| gcmkPRINT("%08X : %08X", |
| GpuAddress, data[0]); |
| break; |
| default: |
| break; |
| } |
| } |
| |
| static void |
| _DumpKernelCommandBuffer( |
| IN gckCOMMAND Command |
| ) |
| { |
| gctINT i; |
| gctUINT64 physical = 0; |
| gctUINT32 address; |
| gctPOINTER entry = gcvNULL; |
| |
| for (i = 0; i < gcdCOMMAND_QUEUES; i++) |
| { |
| entry = Command->queues[i].logical; |
| |
| gckOS_GetPhysicalAddress(Command->os, entry, &physical); |
| |
| gcmkPRINT("Kernel command buffer %d\n", i); |
| |
| gcmkSAFECASTPHYSADDRT(address, physical); |
| |
| _DumpBuffer(entry, address, Command->pageSize); |
| } |
| } |
| |
| /******************************************************************************\ |
| ****************************** gckCOMMAND API Code ****************************** |
| \******************************************************************************/ |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Construct |
| ** |
| ** Construct a new gckCOMMAND object. |
| ** |
| ** INPUT: |
| ** |
| ** gckKERNEL Kernel |
| ** Pointer to an gckKERNEL object. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckCOMMAND * Command |
| ** Pointer to a variable that will hold the pointer to the gckCOMMAND |
| ** object. |
| */ |
| gceSTATUS |
| gckCOMMAND_Construct( |
| IN gckKERNEL Kernel, |
| OUT gckCOMMAND * Command |
| ) |
| { |
| gckOS os; |
| gckCOMMAND command = gcvNULL; |
| gceSTATUS status; |
| gctINT i; |
| gctPOINTER pointer = gcvNULL; |
| gctSIZE_T pageSize; |
| |
| gcmkHEADER_ARG("Kernel=0x%x", Kernel); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); |
| gcmkVERIFY_ARGUMENT(Command != gcvNULL); |
| |
| /* Extract the gckOS object. */ |
| os = Kernel->os; |
| |
| /* Allocate the gckCOMMAND structure. */ |
| gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckCOMMAND), &pointer)); |
| command = pointer; |
| |
| /* Reset the entire object. */ |
| gcmkONERROR(gckOS_ZeroMemory(command, gcmSIZEOF(struct _gckCOMMAND))); |
| |
| /* Initialize the gckCOMMAND object.*/ |
| command->object.type = gcvOBJ_COMMAND; |
| command->kernel = Kernel; |
| command->os = os; |
| |
| /* Get the command buffer requirements. */ |
| gcmkONERROR(gckHARDWARE_QueryCommandBuffer( |
| Kernel->hardware, |
| &command->alignment, |
| &command->reservedHead, |
| &command->reservedTail |
| )); |
| |
| /* Create the command queue mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &command->mutexQueue)); |
| |
| /* Create the context switching mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContext)); |
| |
| #if VIVANTE_PROFILER_CONTEXT |
| /* Create the context switching mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContextSeq)); |
| #endif |
| |
| /* Create the power management semaphore. */ |
| gcmkONERROR(gckOS_CreateSemaphore(os, &command->powerSemaphore)); |
| |
| /* Create the commit atom. */ |
| gcmkONERROR(gckOS_AtomConstruct(os, &command->atomCommit)); |
| |
| /* Get the page size from teh OS. */ |
| gcmkONERROR(gckOS_GetPageSize(os, &pageSize)); |
| |
| gcmkSAFECASTSIZET(command->pageSize, pageSize); |
| |
| /* Get process ID. */ |
| gcmkONERROR(gckOS_GetProcessID(&command->kernelProcessID)); |
| |
| /* Set hardware to pipe 0. */ |
| command->pipeSelect = gcvPIPE_INVALID; |
| |
| /* Pre-allocate the command queues. */ |
| for (i = 0; i < gcdCOMMAND_QUEUES; ++i) |
| { |
| gcmkONERROR(gckOS_AllocateNonPagedMemory( |
| os, |
| gcvFALSE, |
| &pageSize, |
| &command->queues[i].physical, |
| &command->queues[i].logical |
| )); |
| |
| gcmkONERROR(gckHARDWARE_ConvertLogical( |
| Kernel->hardware, |
| command->queues[i].logical, |
| gcvFALSE, |
| &command->queues[i].address |
| )); |
| |
| gcmkONERROR(gckOS_CreateSignal( |
| os, gcvFALSE, &command->queues[i].signal |
| )); |
| |
| gcmkONERROR(gckOS_Signal( |
| os, command->queues[i].signal, gcvTRUE |
| )); |
| } |
| |
| #if gcdRECORD_COMMAND |
| gcmkONERROR(gckRECORDER_Construct(os, Kernel->hardware, &command->recorder)); |
| #endif |
| |
| gcmkONERROR(gckFENCE_Create( |
| os, Kernel, &command->fence |
| )); |
| |
| /* No command queue in use yet. */ |
| command->index = -1; |
| command->logical = gcvNULL; |
| command->newQueue = gcvFALSE; |
| |
| /* Command is not yet running. */ |
| command->running = gcvFALSE; |
| |
| /* Command queue is idle. */ |
| command->idle = gcvTRUE; |
| |
| /* Commit stamp is zero. */ |
| command->commitStamp = 0; |
| |
| /* END event signal not created. */ |
| command->endEventSignal = gcvNULL; |
| |
| command->queue.front = 0; |
| command->queue.rear = 0; |
| command->queue.count = 0; |
| |
| /* Return pointer to the gckCOMMAND object. */ |
| *Command = command; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Command=0x%x", *Command); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Roll back. */ |
| if (command != gcvNULL) |
| { |
| gcmkVERIFY_OK(gckCOMMAND_Destroy(command)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Destroy |
| ** |
| ** Destroy an gckCOMMAND object. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to destroy. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Destroy( |
| IN gckCOMMAND Command |
| ) |
| { |
| gctINT i; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Stop the command queue. */ |
| gcmkVERIFY_OK(gckCOMMAND_Stop(Command, gcvFALSE)); |
| |
| for (i = 0; i < gcdCOMMAND_QUEUES; ++i) |
| { |
| if (Command->queues[i].signal) |
| { |
| gcmkVERIFY_OK(gckOS_DestroySignal( |
| Command->os, Command->queues[i].signal |
| )); |
| } |
| |
| if (Command->queues[i].logical) |
| { |
| gcmkVERIFY_OK(gckOS_FreeNonPagedMemory( |
| Command->os, |
| Command->pageSize, |
| Command->queues[i].physical, |
| Command->queues[i].logical |
| )); |
| } |
| } |
| |
| /* END event signal. */ |
| if (Command->endEventSignal != gcvNULL) |
| { |
| gcmkVERIFY_OK(gckOS_DestroySignal( |
| Command->os, Command->endEventSignal |
| )); |
| } |
| |
| if (Command->mutexContext) |
| { |
| /* Delete the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContext)); |
| } |
| |
| #if VIVANTE_PROFILER_CONTEXT |
| if (Command->mutexContextSeq != gcvNULL) |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContextSeq)); |
| #endif |
| |
| if (Command->mutexQueue) |
| { |
| /* Delete the command queue mutex. */ |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexQueue)); |
| } |
| |
| if (Command->powerSemaphore) |
| { |
| /* Destroy the power management semaphore. */ |
| gcmkVERIFY_OK(gckOS_DestroySemaphore(Command->os, Command->powerSemaphore)); |
| } |
| |
| if (Command->atomCommit) |
| { |
| /* Destroy the commit atom. */ |
| gcmkVERIFY_OK(gckOS_AtomDestroy(Command->os, Command->atomCommit)); |
| } |
| |
| #if gcdSECURE_USER |
| /* Free state array. */ |
| if (Command->hintArrayAllocated) |
| { |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray))); |
| Command->hintArrayAllocated = gcvFALSE; |
| } |
| #endif |
| |
| #if gcdRECORD_COMMAND |
| gckRECORDER_Destory(Command->os, Command->recorder); |
| #endif |
| |
| if (Command->stateMap) |
| { |
| gcmkOS_SAFE_FREE(Command->os, Command->stateMap); |
| } |
| |
| if (Command->fence) |
| { |
| gcmkVERIFY_OK(gckFENCE_Destory(Command->os, Command->fence)); |
| } |
| |
| /* Mark object as unknown. */ |
| Command->object.type = gcvOBJ_UNKNOWN; |
| |
| /* Free the gckCOMMAND object. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, Command)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_EnterCommit |
| ** |
| ** Acquire command queue synchronization objects. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to destroy. |
| ** |
| ** gctBOOL FromPower |
| ** Determines whether the call originates from inside the power |
| ** management or not. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_EnterCommit( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromPower |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| gctBOOL atomIncremented = gcvFALSE; |
| gctBOOL semaAcquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Extract the gckHARDWARE and gckEVENT objects. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| if (!FromPower) |
| { |
| /* Increment COMMIT atom to let power management know that a commit is |
| ** in progress. */ |
| gcmkONERROR(_IncrementCommitAtom(Command, gcvTRUE)); |
| atomIncremented = gcvTRUE; |
| |
| /* Notify the system the GPU has a commit. */ |
| gcmkONERROR(gckOS_Broadcast(Command->os, |
| hardware, |
| gcvBROADCAST_GPU_COMMIT)); |
| |
| /* Acquire the power management semaphore. */ |
| gcmkONERROR(gckOS_AcquireSemaphore(Command->os, |
| Command->powerSemaphore)); |
| semaAcquired = gcvTRUE; |
| } |
| |
| /* Grab the conmmand queue mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(Command->os, |
| Command->mutexQueue, |
| gcvINFINITE)); |
| |
| /* Success. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (semaAcquired) |
| { |
| /* Release the power management semaphore. */ |
| gcmkVERIFY_OK(gckOS_ReleaseSemaphore( |
| Command->os, Command->powerSemaphore |
| )); |
| } |
| |
| if (atomIncremented) |
| { |
| /* Decrement the commit atom. */ |
| gcmkVERIFY_OK(_IncrementCommitAtom( |
| Command, gcvFALSE |
| )); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_ExitCommit |
| ** |
| ** Release command queue synchronization objects. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to destroy. |
| ** |
| ** gctBOOL FromPower |
| ** Determines whether the call originates from inside the power |
| ** management or not. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_ExitCommit( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromPower |
| ) |
| { |
| gceSTATUS status; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Release the power mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); |
| |
| if (!FromPower) |
| { |
| /* Release the power management semaphore. */ |
| gcmkONERROR(gckOS_ReleaseSemaphore(Command->os, |
| Command->powerSemaphore)); |
| |
| /* Decrement the commit atom. */ |
| gcmkONERROR(_IncrementCommitAtom(Command, gcvFALSE)); |
| } |
| |
| /* Success. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Start |
| ** |
| ** Start up the command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to start. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Start( |
| IN gckCOMMAND Command |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| gctUINT32 waitOffset = 0; |
| gctUINT32 waitLinkBytes; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| if (Command->running) |
| { |
| /* Command queue already running. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| /* Extract the gckHARDWARE object. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| if (Command->logical == gcvNULL) |
| { |
| /* Start at beginning of a new queue. */ |
| gcmkONERROR(_NewQueue(Command)); |
| } |
| |
| /* Start at beginning of page. */ |
| Command->offset = 0; |
| |
| /* Set abvailable number of bytes for WAIT/LINK command sequence. */ |
| waitLinkBytes = Command->pageSize; |
| |
| /* Append WAIT/LINK. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| hardware, |
| Command->logical, |
| 0, |
| &waitLinkBytes, |
| &waitOffset, |
| &Command->waitSize |
| )); |
| |
| Command->waitLogical = (gctUINT8_PTR) Command->logical + waitOffset; |
| Command->waitPhysical = Command->physical + waitOffset; |
| |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| /* Flush the cache for the wait/link. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| Command->kernelProcessID, |
| gcvNULL, |
| (gctUINT32)Command->physical, |
| Command->logical, |
| waitLinkBytes |
| )); |
| #endif |
| |
| /* Adjust offset. */ |
| Command->offset = waitLinkBytes; |
| Command->newQueue = gcvFALSE; |
| |
| #if gcdSECURITY |
| /* Start FE by calling security service. */ |
| gckKERNEL_SecurityStartCommand( |
| Command->kernel |
| ); |
| #else |
| /* Enable command processor. */ |
| gcmkONERROR(gckHARDWARE_Execute( |
| hardware, |
| Command->address, |
| waitLinkBytes |
| )); |
| #endif |
| |
| /* Command queue is running. */ |
| Command->running = gcvTRUE; |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Stop |
| ** |
| ** Stop the command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to stop. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Stop( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromRecovery |
| ) |
| { |
| gckHARDWARE hardware; |
| gceSTATUS status; |
| gctUINT32 idle; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| if (!Command->running) |
| { |
| /* Command queue is not running. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| /* Extract the gckHARDWARE object. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| if (gckHARDWARE_IsFeatureAvailable(hardware, |
| gcvFEATURE_END_EVENT) == gcvSTATUS_TRUE) |
| { |
| /* Allocate the signal. */ |
| if (Command->endEventSignal == gcvNULL) |
| { |
| gcmkONERROR(gckOS_CreateSignal(Command->os, |
| gcvTRUE, |
| &Command->endEventSignal)); |
| } |
| |
| /* Append the END EVENT command to trigger the signal. */ |
| gcmkONERROR(gckEVENT_Stop(Command->kernel->eventObj, |
| Command->kernelProcessID, |
| Command->waitPhysical, |
| Command->waitLogical, |
| Command->endEventSignal, |
| &Command->waitSize)); |
| } |
| else |
| { |
| /* Replace last WAIT with END. */ |
| gcmkONERROR(gckHARDWARE_End( |
| hardware, Command->waitLogical, &Command->waitSize |
| )); |
| |
| #if gcdSECURITY |
| gcmkONERROR(gckKERNEL_SecurityExecute( |
| Command->kernel, Command->waitLogical, 8 |
| )); |
| #endif |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail(Command->kernel->hardware, |
| Command->logical, |
| Command->offset)); |
| |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| /* Flush the cache for the END. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| Command->kernelProcessID, |
| gcvNULL, |
| (gctUINT32)Command->waitPhysical, |
| Command->waitLogical, |
| Command->waitSize |
| )); |
| #endif |
| |
| /* Wait for idle. */ |
| gcmkONERROR(gckHARDWARE_GetIdle(hardware, !FromRecovery, &idle)); |
| } |
| |
| /* Command queue is no longer running. */ |
| Command->running = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Commit |
| ** |
| ** Commit a command buffer to the command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** gckCONTEXT Context |
| ** Pointer to a gckCONTEXT object. |
| ** |
| ** gcoCMDBUF CommandBuffer |
| ** Pointer to a gcoCMDBUF object. |
| ** |
| ** gcsSTATE_DELTA_PTR StateDelta |
| ** Pointer to the state delta. |
| ** |
| ** gctUINT32 ProcessID |
| ** Current process ID. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| #if gcdMULTI_GPU |
| gceSTATUS |
| gckCOMMAND_Commit( |
| IN gckCOMMAND Command, |
| IN gckCONTEXT Context, |
| IN gcoCMDBUF CommandBuffer, |
| IN gcsSTATE_DELTA_PTR StateDelta, |
| IN gcsQUEUE_PTR EventQueue, |
| IN gctUINT32 ProcessID, |
| IN gceCORE_3D_MASK ChipEnable |
| ) |
| #else |
| gceSTATUS |
| gckCOMMAND_Commit( |
| IN gckCOMMAND Command, |
| IN gckCONTEXT Context, |
| IN gcoCMDBUF CommandBuffer, |
| IN gcsSTATE_DELTA_PTR StateDelta, |
| IN gcsQUEUE_PTR EventQueue, |
| IN gctUINT32 ProcessID |
| ) |
| #endif |
| { |
| gceSTATUS status; |
| gctBOOL commitEntered = gcvFALSE; |
| gctBOOL contextAcquired = gcvFALSE; |
| gckHARDWARE hardware; |
| gctBOOL needCopy = gcvFALSE; |
| gcsQUEUE_PTR eventRecord = gcvNULL; |
| gcsQUEUE _eventRecord; |
| gcsQUEUE_PTR nextEventRecord; |
| gctBOOL commandBufferMapped = gcvFALSE; |
| gcoCMDBUF commandBufferObject = gcvNULL; |
| gctBOOL stall = gcvFALSE; |
| |
| #if !gcdNULL_DRIVER |
| gcsCONTEXT_PTR contextBuffer; |
| struct _gcoCMDBUF _commandBufferObject; |
| gctPHYS_ADDR_T commandBufferPhysical; |
| gctUINT8_PTR commandBufferLogical = gcvNULL; |
| gctUINT32 commandBufferAddress = 0; |
| gctUINT8_PTR commandBufferLink = gcvNULL; |
| gctUINT commandBufferSize; |
| gctSIZE_T nopBytes; |
| gctUINT32 pipeBytes; |
| gctUINT32 linkBytes; |
| gctSIZE_T bytes; |
| gctUINT32 offset; |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| gctPHYS_ADDR entryPhysical; |
| #endif |
| gctPOINTER entryLogical; |
| gctUINT32 entryAddress; |
| gctUINT32 entryBytes; |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| gctPHYS_ADDR exitPhysical; |
| #endif |
| gctPOINTER exitLogical; |
| gctUINT32 exitAddress; |
| gctUINT32 exitBytes; |
| gctUINT32 waitLinkPhysical; |
| gctPOINTER waitLinkLogical; |
| gctUINT32 waitLinkAddress; |
| gctUINT32 waitLinkBytes; |
| gctUINT32 waitPhysical; |
| gctPOINTER waitLogical; |
| gctUINT32 waitOffset; |
| gctUINT32 waitSize; |
| |
| #ifdef __QNXNTO__ |
| gctPOINTER userCommandBufferLogical = gcvNULL; |
| gctBOOL userCommandBufferLogicalMapped = gcvFALSE; |
| gctPOINTER userCommandBufferLink = gcvNULL; |
| gctBOOL userCommandBufferLinkMapped = gcvFALSE; |
| #endif |
| |
| #if gcdPROCESS_ADDRESS_SPACE |
| gctSIZE_T mmuConfigureBytes; |
| gctPOINTER mmuConfigureLogical = gcvNULL; |
| gctUINT32 mmuConfigureAddress; |
| gctPOINTER mmuConfigurePhysical = 0; |
| gctSIZE_T mmuConfigureWaitLinkOffset; |
| gckMMU mmu; |
| gctSIZE_T reservedBytes; |
| gctUINT32 oldValue; |
| #endif |
| |
| #if gcdDUMP_COMMAND |
| gctPOINTER contextDumpLogical = gcvNULL; |
| gctSIZE_T contextDumpBytes = 0; |
| gctPOINTER bufferDumpLogical = gcvNULL; |
| gctSIZE_T bufferDumpBytes = 0; |
| # endif |
| #endif |
| |
| #if VIVANTE_PROFILER_CONTEXT |
| gctBOOL sequenceAcquired = gcvFALSE; |
| #endif |
| |
| gctPOINTER pointer = gcvNULL; |
| |
| #if gcdMULTI_GPU |
| gctSIZE_T chipEnableBytes; |
| #endif |
| |
| gctUINT32 exitLinkLow = 0, exitLinkHigh = 0; |
| gctUINT32 entryLinkLow = 0, entryLinkHigh = 0; |
| gctUINT32 commandLinkLow = 0, commandLinkHigh = 0; |
| |
| gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer = gcvNULL; |
| |
| gcmkHEADER_ARG( |
| "Command=0x%x CommandBuffer=0x%x ProcessID=%d", |
| Command, CommandBuffer, ProcessID |
| ); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| if (Command->kernel->hardware->type== gcvHARDWARE_2D) |
| { |
| /* There is no context for 2D. */ |
| Context = gcvNULL; |
| } |
| |
| #if gcdPROCESS_ADDRESS_SPACE |
| gcmkONERROR(gckKERNEL_GetProcessMMU(Command->kernel, &mmu)); |
| |
| gcmkONERROR(gckOS_AtomicExchange(Command->os, |
| mmu->pageTableDirty[Command->kernel->core], |
| 0, |
| &oldValue)); |
| #else |
| #endif |
| |
| #if VIVANTE_PROFILER_CONTEXT |
| if((Command->kernel->hardware->gpuProfiler) && (Command->kernel->profileEnable)) |
| { |
| /* Acquire the context sequnence mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, Command->mutexContextSeq, gcvINFINITE |
| )); |
| sequenceAcquired = gcvTRUE; |
| } |
| #endif |
| |
| /* Acquire the command queue. */ |
| gcmkONERROR(gckCOMMAND_EnterCommit(Command, gcvFALSE)); |
| commitEntered = gcvTRUE; |
| |
| /* Acquire the context switching mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, Command->mutexContext, gcvINFINITE |
| )); |
| contextAcquired = gcvTRUE; |
| |
| /* Extract the gckHARDWARE and gckEVENT objects. */ |
| hardware = Command->kernel->hardware; |
| |
| /* Check wehther we need to copy the structures or not. */ |
| gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy)); |
| |
| #if gcdNULL_DRIVER |
| /* Context switch required? */ |
| if ((Context != gcvNULL) && (Command->currContext != Context)) |
| { |
| /* Yes, merge in the deltas. */ |
| gckCONTEXT_Update(Context, ProcessID, StateDelta); |
| |
| /* Update the current context. */ |
| Command->currContext = Context; |
| } |
| #else |
| if (needCopy) |
| { |
| commandBufferObject = &_commandBufferObject; |
| |
| gcmkONERROR(gckOS_CopyFromUserData( |
| Command->os, |
| commandBufferObject, |
| CommandBuffer, |
| gcmSIZEOF(struct _gcoCMDBUF) |
| )); |
| |
| gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER); |
| } |
| else |
| { |
| gcmkONERROR(gckOS_MapUserPointer( |
| Command->os, |
| CommandBuffer, |
| gcmSIZEOF(struct _gcoCMDBUF), |
| &pointer |
| )); |
| |
| commandBufferObject = pointer; |
| |
| gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER); |
| commandBufferMapped = gcvTRUE; |
| } |
| |
| /* Query the size of NOP command. */ |
| gcmkONERROR(gckHARDWARE_Nop( |
| hardware, gcvNULL, &nopBytes |
| )); |
| |
| /* Query the size of pipe select command sequence. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| hardware, gcvNULL, gcvPIPE_3D, &pipeBytes |
| )); |
| |
| /* Query the size of LINK command. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, gcvNULL, 0, 0, &linkBytes, gcvNULL, gcvNULL |
| )); |
| |
| #if gcdMULTI_GPU |
| /* Query the size of chip enable command sequence. */ |
| gcmkONERROR(gckHARDWARE_ChipEnable( |
| hardware, gcvNULL, 0, &chipEnableBytes |
| )); |
| #endif |
| |
| /* Compute the command buffer entry and the size. */ |
| commandBufferLogical |
| = (gctUINT8_PTR) gcmUINT64_TO_PTR(commandBufferObject->logical) |
| + commandBufferObject->startOffset; |
| |
| /* Get the hardware address. */ |
| if (Command->kernel->virtualCommandBuffer) |
| { |
| gckKERNEL kernel = Command->kernel; |
| |
| virtualCommandBuffer = gcmNAME_TO_PTR(commandBufferObject->physical); |
| |
| if (virtualCommandBuffer == gcvNULL) |
| { |
| gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); |
| } |
| |
| gcmkONERROR(gckKERNEL_GetGPUAddress( |
| Command->kernel, |
| commandBufferLogical, |
| gcvTRUE, |
| virtualCommandBuffer, |
| &commandBufferAddress |
| )); |
| } |
| else |
| { |
| gcmkONERROR(gckHARDWARE_ConvertLogical( |
| hardware, |
| commandBufferLogical, |
| gcvTRUE, |
| &commandBufferAddress |
| )); |
| } |
| |
| /* Get the physical address. */ |
| gcmkONERROR(gckOS_UserLogicalToPhysical( |
| Command->os, |
| commandBufferLogical, |
| &commandBufferPhysical |
| )); |
| |
| #ifdef __QNXNTO__ |
| userCommandBufferLogical = (gctPOINTER) commandBufferLogical; |
| |
| gcmkONERROR(gckOS_MapUserPointer( |
| Command->os, |
| userCommandBufferLogical, |
| 0, |
| &pointer)); |
| |
| commandBufferLogical = pointer; |
| |
| userCommandBufferLogicalMapped = gcvTRUE; |
| #endif |
| |
| commandBufferSize |
| = commandBufferObject->offset |
| + Command->reservedTail |
| - commandBufferObject->startOffset; |
| |
| gcmkONERROR(_FlushMMU(Command)); |
| |
| if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FE_NEED_DUMMYDRAW) && Command->currContext == gcvNULL) |
| { |
| gcmkONERROR(_DummyDraw(Command)); |
| } |
| |
| /* Get the current offset. */ |
| offset = Command->offset; |
| |
| /* Compute number of bytes left in current kernel command queue. */ |
| bytes = Command->pageSize - offset; |
| |
| #if gcdMULTI_GPU |
| if (Command->kernel->core == gcvCORE_MAJOR) |
| { |
| commandBufferSize += chipEnableBytes; |
| |
| gcmkONERROR(gckHARDWARE_ChipEnable( |
| hardware, |
| commandBufferLogical + pipeBytes, |
| ChipEnable, |
| &chipEnableBytes |
| )); |
| |
| gcmkONERROR(gckHARDWARE_ChipEnable( |
| hardware, |
| commandBufferLogical + commandBufferSize - linkBytes - chipEnableBytes, |
| gcvCORE_3D_ALL_MASK, |
| &chipEnableBytes |
| )); |
| } |
| else |
| { |
| commandBufferSize += nopBytes; |
| |
| gcmkONERROR(gckHARDWARE_Nop( |
| hardware, |
| commandBufferLogical + pipeBytes, |
| &nopBytes |
| )); |
| |
| gcmkONERROR(gckHARDWARE_Nop( |
| hardware, |
| commandBufferLogical + commandBufferSize - linkBytes - nopBytes, |
| &nopBytes |
| )); |
| } |
| #endif |
| |
| /* Query the size of WAIT/LINK command sequence. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| hardware, |
| gcvNULL, |
| offset, |
| &waitLinkBytes, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| /* Is there enough space in the current command queue? */ |
| if (bytes < waitLinkBytes) |
| { |
| /* No, create a new one. */ |
| gcmkONERROR(_NewQueue(Command)); |
| |
| /* Get the new current offset. */ |
| offset = Command->offset; |
| |
| /* Recompute the number of bytes in the new kernel command queue. */ |
| bytes = Command->pageSize - offset; |
| gcmkASSERT(bytes >= waitLinkBytes); |
| } |
| |
| /* Compute the location if WAIT/LINK command sequence. */ |
| waitLinkPhysical = Command->physical + offset; |
| waitLinkLogical = (gctUINT8_PTR) Command->logical + offset; |
| waitLinkAddress = Command->address + offset; |
| |
| /* Context switch required? */ |
| if (Context == gcvNULL) |
| { |
| /* See if we have to switch pipes for the command buffer. */ |
| if (commandBufferObject->entryPipe == Command->pipeSelect) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the entry command buffer pipes |
| ** are different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* Compute the entry. */ |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) commandBufferPhysical + offset; |
| #endif |
| entryLogical = commandBufferLogical + offset; |
| entryAddress = commandBufferAddress + offset; |
| entryBytes = commandBufferSize - offset; |
| |
| Command->currContext = gcvNULL; |
| } |
| else if (Command->currContext != Context) |
| { |
| /* Temporary disable context length oprimization. */ |
| Context->dirty = gcvTRUE; |
| |
| /* Get the current context buffer. */ |
| contextBuffer = Context->buffer; |
| |
| /* Yes, merge in the deltas. */ |
| gcmkONERROR(gckCONTEXT_Update(Context, ProcessID, StateDelta)); |
| |
| /* Determine context entry and exit points. */ |
| if (0) |
| { |
| /* Reset 2D dirty flag. */ |
| Context->dirty2D = gcvFALSE; |
| |
| if (Context->dirty || commandBufferObject->using3D) |
| { |
| /*************************************************************** |
| ** SWITCHING CONTEXT: 2D and 3D are used. |
| */ |
| |
| /* Reset 3D dirty flag. */ |
| Context->dirty3D = gcvFALSE; |
| |
| /* Compute the entry. */ |
| if (Command->pipeSelect == gcvPIPE_2D) |
| { |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) contextBuffer->physical + pipeBytes; |
| #endif |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical + pipeBytes; |
| entryAddress = contextBuffer->address + pipeBytes; |
| entryBytes = Context->bufferSize - pipeBytes; |
| } |
| else |
| { |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) contextBuffer->physical; |
| #endif |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical; |
| entryAddress = contextBuffer->address; |
| entryBytes = Context->bufferSize; |
| } |
| |
| /* See if we have to switch pipes between the context |
| and command buffers. */ |
| if (commandBufferObject->entryPipe == gcvPIPE_3D) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the initial context pipes are |
| different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* Ensure the NOP between 2D and 3D is in place so that the |
| execution falls through from 2D to 3D. */ |
| gcmkONERROR(gckHARDWARE_Nop( |
| hardware, |
| contextBuffer->link2D, |
| &nopBytes |
| )); |
| |
| /* Generate a LINK from the context buffer to |
| the command buffer. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| contextBuffer->link3D, |
| commandBufferAddress + offset, |
| commandBufferSize - offset, |
| &linkBytes, |
| &commandLinkLow, |
| &commandLinkHigh |
| )); |
| |
| /* Mark context as not dirty. */ |
| Context->dirty = gcvFALSE; |
| } |
| else |
| { |
| /*************************************************************** |
| ** SWITCHING CONTEXT: 2D only command buffer. |
| */ |
| |
| /* Mark 3D as dirty. */ |
| Context->dirty3D = gcvTRUE; |
| |
| /* Compute the entry. */ |
| if (Command->pipeSelect == gcvPIPE_2D) |
| { |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) contextBuffer->physical + pipeBytes; |
| #endif |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical + pipeBytes; |
| entryAddress = contextBuffer->address + pipeBytes; |
| entryBytes = Context->entryOffset3D - pipeBytes; |
| } |
| else |
| { |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) contextBuffer->physical; |
| #endif |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical; |
| entryAddress = contextBuffer->address; |
| entryBytes = Context->entryOffset3D; |
| } |
| |
| /* Store the current context buffer. */ |
| Context->dirtyBuffer = contextBuffer; |
| |
| /* See if we have to switch pipes between the context |
| and command buffers. */ |
| if (commandBufferObject->entryPipe == gcvPIPE_2D) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the initial context pipes are |
| different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* 3D is not used, generate a LINK from the end of 2D part of |
| the context buffer to the command buffer. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| contextBuffer->link2D, |
| commandBufferAddress + offset, |
| commandBufferSize - offset, |
| &linkBytes, |
| &commandLinkLow, |
| &commandLinkHigh |
| )); |
| } |
| } |
| |
| /* Not using 2D. */ |
| else |
| { |
| |
| /* Store the current context buffer. */ |
| Context->dirtyBuffer = contextBuffer; |
| |
| if (Context->dirty || commandBufferObject->using3D) |
| { |
| /*************************************************************** |
| ** SWITCHING CONTEXT: 3D only command buffer. |
| */ |
| |
| /* Reset 3D dirty flag. */ |
| Context->dirty3D = gcvFALSE; |
| |
| /* Determine context buffer entry offset. */ |
| offset = (Command->pipeSelect == gcvPIPE_3D) |
| |
| /* Skip pipe switching sequence. */ |
| ? Context->entryOffset3D + Context->pipeSelectBytes |
| |
| /* Do not skip pipe switching sequence. */ |
| : Context->entryOffset3D; |
| |
| /* Compute the entry. */ |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) contextBuffer->physical + offset; |
| #endif |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical + offset; |
| entryAddress = contextBuffer->address + offset; |
| entryBytes = Context->bufferSize - offset; |
| |
| /* See if we have to switch pipes between the context |
| and command buffers. */ |
| if (commandBufferObject->entryPipe == gcvPIPE_3D) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the initial context pipes are |
| different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* Generate a LINK from the context buffer to |
| the command buffer. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| contextBuffer->link3D, |
| commandBufferAddress + offset, |
| commandBufferSize - offset, |
| &linkBytes, |
| &commandLinkLow, |
| &commandLinkHigh |
| )); |
| } |
| else |
| { |
| /*************************************************************** |
| ** SWITCHING CONTEXT: "XD" command buffer - neither 2D nor 3D. |
| */ |
| |
| /* Mark 3D as dirty. */ |
| Context->dirty3D = gcvTRUE; |
| |
| /* Compute the entry. */ |
| if (Command->pipeSelect == gcvPIPE_3D) |
| { |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical |
| = (gctUINT8_PTR) contextBuffer->physical |
| + Context->entryOffsetXDFrom3D; |
| #endif |
| entryLogical |
| = (gctUINT8_PTR) contextBuffer->logical |
| + Context->entryOffsetXDFrom3D; |
| |
| entryAddress |
| = contextBuffer->address |
| + Context->entryOffsetXDFrom3D; |
| |
| entryBytes |
| = Context->bufferSize |
| - Context->entryOffsetXDFrom3D; |
| } |
| else |
| { |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical |
| = (gctUINT8_PTR) contextBuffer->physical |
| + Context->entryOffsetXDFrom2D; |
| #endif |
| entryLogical |
| = (gctUINT8_PTR) contextBuffer->logical |
| + Context->entryOffsetXDFrom2D; |
| |
| entryAddress |
| = contextBuffer->address |
| + Context->entryOffsetXDFrom2D; |
| |
| entryBytes |
| = Context->totalSize |
| - Context->entryOffsetXDFrom2D; |
| } |
| |
| /* See if we have to switch pipes between the context |
| and command buffers. */ |
| if (commandBufferObject->entryPipe == gcvPIPE_3D) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the initial context pipes are |
| different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* Generate a LINK from the context buffer to |
| the command buffer. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| contextBuffer->link3D, |
| commandBufferAddress + offset, |
| commandBufferSize - offset, |
| &linkBytes, |
| &commandLinkLow, |
| &commandLinkHigh |
| )); |
| } |
| } |
| |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| /* Flush the context buffer cache. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| Command->kernelProcessID, |
| gcvNULL, |
| (gctUINT32)entryPhysical, |
| entryLogical, |
| entryBytes |
| )); |
| #endif |
| |
| /* Update the current context. */ |
| Command->currContext = Context; |
| |
| #if gcdDUMP_COMMAND |
| contextDumpLogical = entryLogical; |
| contextDumpBytes = entryBytes; |
| #endif |
| |
| #if gcdSECURITY |
| /* Commit context buffer to trust zone. */ |
| gckKERNEL_SecurityExecute( |
| Command->kernel, |
| entryLogical, |
| entryBytes - 8 |
| ); |
| #endif |
| |
| #if gcdRECORD_COMMAND |
| gckRECORDER_Record( |
| Command->recorder, |
| gcvNULL, |
| 0xFFFFFFFF, |
| entryLogical, |
| entryBytes - 8 |
| ); |
| #endif |
| } |
| |
| /* Same context. */ |
| else |
| { |
| /* Determine context entry and exit points. */ |
| if (commandBufferObject->using2D && Context->dirty2D) |
| { |
| /* Reset 2D dirty flag. */ |
| Context->dirty2D = gcvFALSE; |
| |
| /* Get the "dirty" context buffer. */ |
| contextBuffer = Context->dirtyBuffer; |
| |
| if (commandBufferObject->using3D && Context->dirty3D) |
| { |
| /* Reset 3D dirty flag. */ |
| Context->dirty3D = gcvFALSE; |
| |
| /* Compute the entry. */ |
| if (Command->pipeSelect == gcvPIPE_2D) |
| { |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) contextBuffer->physical + pipeBytes; |
| #endif |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical + pipeBytes; |
| entryAddress = contextBuffer->address + pipeBytes; |
| entryBytes = Context->bufferSize - pipeBytes; |
| } |
| else |
| { |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) contextBuffer->physical; |
| #endif |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical; |
| entryAddress = contextBuffer->address; |
| entryBytes = Context->bufferSize; |
| } |
| |
| /* See if we have to switch pipes between the context |
| and command buffers. */ |
| if (commandBufferObject->entryPipe == gcvPIPE_3D) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the initial context pipes are |
| different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* Ensure the NOP between 2D and 3D is in place so that the |
| execution falls through from 2D to 3D. */ |
| gcmkONERROR(gckHARDWARE_Nop( |
| hardware, |
| contextBuffer->link2D, |
| &nopBytes |
| )); |
| |
| /* Generate a LINK from the context buffer to |
| the command buffer. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| contextBuffer->link3D, |
| commandBufferAddress + offset, |
| commandBufferSize - offset, |
| &linkBytes, |
| &commandLinkLow, |
| &commandLinkHigh |
| )); |
| } |
| else |
| { |
| /* Compute the entry. */ |
| if (Command->pipeSelect == gcvPIPE_2D) |
| { |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) contextBuffer->physical + pipeBytes; |
| #endif |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical + pipeBytes; |
| entryAddress = contextBuffer->address + pipeBytes; |
| entryBytes = Context->entryOffset3D - pipeBytes; |
| } |
| else |
| { |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) contextBuffer->physical; |
| #endif |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical; |
| entryAddress = contextBuffer->address; |
| entryBytes = Context->entryOffset3D; |
| } |
| |
| /* See if we have to switch pipes between the context |
| and command buffers. */ |
| if (commandBufferObject->entryPipe == gcvPIPE_2D) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the initial context pipes are |
| different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* 3D is not used, generate a LINK from the end of 2D part of |
| the context buffer to the command buffer. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| contextBuffer->link2D, |
| commandBufferAddress + offset, |
| commandBufferSize - offset, |
| &linkBytes, |
| &commandLinkLow, |
| &commandLinkHigh |
| )); |
| } |
| } |
| else |
| { |
| if (commandBufferObject->using3D && Context->dirty3D) |
| { |
| /* Reset 3D dirty flag. */ |
| Context->dirty3D = gcvFALSE; |
| |
| /* Get the "dirty" context buffer. */ |
| contextBuffer = Context->dirtyBuffer; |
| |
| /* Determine context buffer entry offset. */ |
| offset = (Command->pipeSelect == gcvPIPE_3D) |
| |
| /* Skip pipe switching sequence. */ |
| ? Context->entryOffset3D + pipeBytes |
| |
| /* Do not skip pipe switching sequence. */ |
| : Context->entryOffset3D; |
| |
| /* Compute the entry. */ |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) contextBuffer->physical + offset; |
| #endif |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical + offset; |
| entryAddress = contextBuffer->address + offset; |
| entryBytes = Context->bufferSize - offset; |
| |
| /* See if we have to switch pipes between the context |
| and command buffers. */ |
| if (commandBufferObject->entryPipe == gcvPIPE_3D) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the initial context pipes are |
| different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* Generate a LINK from the context buffer to |
| the command buffer. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| contextBuffer->link3D, |
| commandBufferAddress + offset, |
| commandBufferSize - offset, |
| &linkBytes, |
| &commandLinkLow, |
| &commandLinkHigh |
| )); |
| } |
| else |
| { |
| /* See if we have to switch pipes for the command buffer. */ |
| if (commandBufferObject->entryPipe == Command->pipeSelect) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the entry command buffer pipes |
| ** are different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* Compute the entry. */ |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| entryPhysical = (gctUINT8_PTR) commandBufferPhysical + offset; |
| #endif |
| entryLogical = commandBufferLogical + offset; |
| entryAddress = commandBufferAddress + offset; |
| entryBytes = commandBufferSize - offset; |
| } |
| } |
| } |
| |
| #if gcdDUMP_COMMAND |
| bufferDumpLogical = commandBufferLogical + offset; |
| bufferDumpBytes = commandBufferSize - offset; |
| #endif |
| |
| #if gcdSECURE_USER |
| /* Process user hints. */ |
| gcmkONERROR(_ProcessHints(Command, ProcessID, commandBufferObject)); |
| #endif |
| |
| /* Determine the location to jump to for the command buffer being |
| ** scheduled. */ |
| if (Command->newQueue) |
| { |
| /* New command queue, jump to the beginning of it. */ |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| exitPhysical = Command->physical; |
| #endif |
| |
| exitLogical = Command->logical; |
| exitAddress = Command->address; |
| exitBytes = Command->offset + waitLinkBytes; |
| } |
| else |
| { |
| /* Still within the preexisting command queue, jump to the new |
| WAIT/LINK command sequence. */ |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| exitPhysical = waitLinkPhysical; |
| #endif |
| exitLogical = waitLinkLogical; |
| exitAddress = waitLinkAddress; |
| exitBytes = waitLinkBytes; |
| } |
| |
| /* Add a new WAIT/LINK command sequence. When the command buffer which is |
| currently being scheduled is fully executed by the GPU, the FE will |
| jump to this WAIT/LINK sequence. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| hardware, |
| waitLinkLogical, |
| offset, |
| &waitLinkBytes, |
| &waitOffset, |
| &waitSize |
| )); |
| |
| /* Compute the location if WAIT command. */ |
| waitPhysical = waitLinkPhysical + waitOffset; |
| waitLogical = (gctUINT8_PTR) waitLinkLogical + waitOffset; |
| |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| /* Flush the command queue cache. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| Command->kernelProcessID, |
| gcvNULL, |
| (gctUINT32)exitPhysical, |
| exitLogical, |
| exitBytes |
| )); |
| #endif |
| |
| /* Determine the location of the LINK command in the command buffer. */ |
| commandBufferLink |
| = (gctUINT8_PTR) gcmUINT64_TO_PTR(commandBufferObject->logical) |
| + commandBufferObject->offset; |
| |
| #ifdef __QNXNTO__ |
| userCommandBufferLink = (gctPOINTER) commandBufferLink; |
| |
| gcmkONERROR(gckOS_MapUserPointer( |
| Command->os, |
| userCommandBufferLink, |
| 0, |
| &pointer)); |
| |
| commandBufferLink = pointer; |
| |
| userCommandBufferLinkMapped = gcvTRUE; |
| #endif |
| |
| #if gcdMULTI_GPU |
| if (Command->kernel->core == gcvCORE_MAJOR) |
| { |
| commandBufferLink += chipEnableBytes; |
| } |
| else |
| { |
| commandBufferLink += nopBytes; |
| } |
| #endif |
| |
| /* Generate a LINK from the end of the command buffer being scheduled |
| back to the kernel command queue. */ |
| #if !gcdSECURITY |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| commandBufferLink, |
| exitAddress, |
| exitBytes, |
| &linkBytes, |
| &exitLinkLow, |
| &exitLinkHigh |
| )); |
| #endif |
| |
| #ifdef __QNXNTO__ |
| gcmkONERROR(gckOS_UnmapUserPointer( |
| Command->os, |
| userCommandBufferLink, |
| 0, |
| commandBufferLink)); |
| |
| userCommandBufferLinkMapped = gcvFALSE; |
| #endif |
| |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| /* Flush the command buffer cache. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| ProcessID, |
| gcvNULL, |
| (gctUINT32)commandBufferPhysical, |
| commandBufferLogical, |
| commandBufferSize |
| )); |
| #endif |
| |
| #if gcdRECORD_COMMAND |
| gckRECORDER_Record( |
| Command->recorder, |
| commandBufferLogical + offset, |
| commandBufferSize - offset - 8, |
| gcvNULL, |
| 0xFFFFFFFF |
| ); |
| |
| gckRECORDER_AdvanceIndex(Command->recorder, Command->commitStamp); |
| #endif |
| |
| #if gcdSECURITY |
| /* Submit command buffer to trust zone. */ |
| gckKERNEL_SecurityExecute( |
| Command->kernel, |
| commandBufferLogical + offset, |
| commandBufferSize - offset - 8 |
| ); |
| #else |
| /* Generate a LINK from the previous WAIT/LINK command sequence to the |
| entry determined above (either the context or the command buffer). |
| This LINK replaces the WAIT instruction from the previous WAIT/LINK |
| pair, therefore we use WAIT metrics for generation of this LINK. |
| This action will execute the entire sequence. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| Command->waitLogical, |
| entryAddress, |
| entryBytes, |
| &Command->waitSize, |
| &entryLinkLow, |
| &entryLinkHigh |
| )); |
| #endif |
| |
| #if gcdLINK_QUEUE_SIZE |
| if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_USER_COMMAND) |
| { |
| gckLINKQUEUE_Enqueue( |
| &hardware->linkQueue, |
| entryAddress, |
| entryAddress + entryBytes, |
| entryLinkLow, |
| entryLinkHigh |
| ); |
| |
| if (commandBufferAddress + offset != entryAddress) |
| { |
| gckLINKQUEUE_Enqueue( |
| &hardware->linkQueue, |
| commandBufferAddress + offset, |
| commandBufferAddress + commandBufferSize, |
| commandLinkLow, |
| commandLinkHigh |
| ); |
| } |
| |
| if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) |
| { |
| /* Dump kernel command.*/ |
| gckLINKQUEUE_Enqueue( |
| &hardware->linkQueue, |
| exitAddress, |
| exitAddress + exitBytes, |
| exitLinkLow, |
| exitLinkHigh |
| ); |
| } |
| } |
| #endif |
| |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| /* Flush the cache for the link. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| Command->kernelProcessID, |
| gcvNULL, |
| (gctUINT32)Command->waitPhysical, |
| Command->waitLogical, |
| Command->waitSize |
| )); |
| #endif |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| Command->waitLogical, |
| Command->waitSize, |
| gceDUMP_BUFFER_LINK, |
| gcvFALSE |
| ); |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| contextDumpLogical, |
| contextDumpBytes, |
| gceDUMP_BUFFER_CONTEXT, |
| gcvFALSE |
| ); |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| bufferDumpLogical, |
| bufferDumpBytes, |
| gceDUMP_BUFFER_USER, |
| gcvFALSE |
| ); |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| waitLinkLogical, |
| waitLinkBytes, |
| gceDUMP_BUFFER_WAITLINK, |
| gcvFALSE |
| ); |
| |
| /* Update the current pipe. */ |
| Command->pipeSelect = commandBufferObject->exitPipe; |
| |
| /* Update command queue offset. */ |
| Command->offset += waitLinkBytes; |
| Command->newQueue = gcvFALSE; |
| |
| /* Update address of last WAIT. */ |
| Command->waitPhysical = waitPhysical; |
| Command->waitLogical = waitLogical; |
| Command->waitSize = waitSize; |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail( |
| hardware, Command->logical, Command->offset |
| )); |
| |
| #if gcdDUMP_COMMAND |
| gcmkPRINT("@[kernel.commit]"); |
| #endif |
| #endif /* gcdNULL_DRIVER */ |
| |
| /* Release the context switching mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| contextAcquired = gcvFALSE; |
| |
| Command->commitStamp++; |
| |
| stall = gcvFALSE; |
| |
| #if gcdLINK_QUEUE_SIZE |
| if (Command->kernel->stuckDump == gcvSTUCK_DUMP_STALL_COMMAND) |
| { |
| if ((Command->commitStamp % (gcdLINK_QUEUE_SIZE/2)) == 0) |
| { |
| /* If only context buffer and command buffer is recorded, |
| ** each commit costs 2 slot in queue, to make sure command |
| ** causing stuck is recorded, number of pending command buffer |
| ** is limited to (gckLINK_QUEUE_SIZE/2) |
| */ |
| stall = gcvTRUE; |
| } |
| } |
| #endif |
| |
| /* Release the command queue. */ |
| gcmkONERROR(gckCOMMAND_ExitCommit(Command, gcvFALSE)); |
| commitEntered = gcvFALSE; |
| |
| if (stall) |
| { |
| #if gcdMULTI_GPU |
| gcmkONERROR(gckCOMMAND_Stall(Command, gcvFALSE, ChipEnable)); |
| #else |
| gcmkONERROR(gckCOMMAND_Stall(Command, gcvFALSE)); |
| #endif |
| } |
| |
| #if VIVANTE_PROFILER_CONTEXT |
| if(sequenceAcquired) |
| { |
| #if gcdMULTI_GPU |
| gcmkONERROR(gckCOMMAND_Stall(Command, gcvTRUE, ChipEnable)); |
| #else |
| gcmkONERROR(gckCOMMAND_Stall(Command, gcvTRUE)); |
| #endif |
| if (Command->currContext) |
| { |
| gcmkONERROR(gckHARDWARE_UpdateContextProfile( |
| hardware, |
| Command->currContext)); |
| } |
| |
| /* Release the context switching mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContextSeq)); |
| sequenceAcquired = gcvFALSE; |
| } |
| #endif |
| |
| /* Loop while there are records in the queue. */ |
| while (EventQueue != gcvNULL) |
| { |
| if (needCopy) |
| { |
| /* Point to stack record. */ |
| eventRecord = &_eventRecord; |
| |
| /* Copy the data from the client. */ |
| gcmkONERROR(gckOS_CopyFromUserData( |
| Command->os, eventRecord, EventQueue, gcmSIZEOF(gcsQUEUE) |
| )); |
| } |
| else |
| { |
| /* Map record into kernel memory. */ |
| gcmkONERROR(gckOS_MapUserPointer(Command->os, |
| EventQueue, |
| gcmSIZEOF(gcsQUEUE), |
| &pointer)); |
| |
| eventRecord = pointer; |
| } |
| |
| /* Append event record to event queue. */ |
| gcmkONERROR(gckEVENT_AddList( |
| Command->kernel->eventObj, &eventRecord->iface, gcvKERNEL_PIXEL, gcvTRUE, gcvFALSE |
| )); |
| |
| /* Next record in the queue. */ |
| nextEventRecord = gcmUINT64_TO_PTR(eventRecord->next); |
| |
| if (!needCopy) |
| { |
| /* Unmap record from kernel memory. */ |
| gcmkONERROR(gckOS_UnmapUserPointer( |
| Command->os, EventQueue, gcmSIZEOF(gcsQUEUE), (gctPOINTER *) eventRecord |
| )); |
| |
| eventRecord = gcvNULL; |
| } |
| |
| EventQueue = nextEventRecord; |
| } |
| |
| if (Command->kernel->eventObj->queueHead == gcvNULL |
| && Command->kernel->hardware->powerManagement == gcvTRUE |
| ) |
| { |
| /* Commit done event by which work thread knows all jobs done. */ |
| gcmkVERIFY_OK( |
| gckEVENT_CommitDone(Command->kernel->eventObj, gcvKERNEL_PIXEL)); |
| } |
| |
| /* Submit events. */ |
| #if gcdMULTI_GPU |
| status = gckEVENT_Submit(Command->kernel->eventObj, gcvTRUE, gcvFALSE, ChipEnable); |
| #else |
| status = gckEVENT_Submit(Command->kernel->eventObj, gcvTRUE, gcvFALSE); |
| #endif |
| if (status == gcvSTATUS_INTERRUPTED) |
| { |
| gcmkTRACE( |
| gcvLEVEL_INFO, |
| "%s(%d): Intterupted in gckEVENT_Submit", |
| __FUNCTION__, __LINE__ |
| ); |
| status = gcvSTATUS_OK; |
| } |
| else |
| { |
| gcmkONERROR(status); |
| } |
| |
| #ifdef __QNXNTO__ |
| if (userCommandBufferLogicalMapped) |
| { |
| gcmkONERROR(gckOS_UnmapUserPointer( |
| Command->os, |
| userCommandBufferLogical, |
| 0, |
| commandBufferLogical)); |
| |
| userCommandBufferLogicalMapped = gcvFALSE; |
| } |
| #endif |
| |
| /* Unmap the command buffer pointer. */ |
| if (commandBufferMapped) |
| { |
| gcmkONERROR(gckOS_UnmapUserPointer( |
| Command->os, |
| CommandBuffer, |
| gcmSIZEOF(struct _gcoCMDBUF), |
| commandBufferObject |
| )); |
| |
| commandBufferMapped = gcvFALSE; |
| } |
| |
| /* Return status. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if ((eventRecord != gcvNULL) && !needCopy) |
| { |
| /* Roll back. */ |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| EventQueue, |
| gcmSIZEOF(gcsQUEUE), |
| (gctPOINTER *) eventRecord |
| )); |
| } |
| |
| if (contextAcquired) |
| { |
| /* Release the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| } |
| |
| if (commitEntered) |
| { |
| /* Release the command queue mutex. */ |
| gcmkVERIFY_OK(gckCOMMAND_ExitCommit(Command, gcvFALSE)); |
| } |
| |
| #if VIVANTE_PROFILER_CONTEXT |
| if (sequenceAcquired) |
| { |
| /* Release the context sequence mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContextSeq)); |
| } |
| #endif |
| |
| #ifdef __QNXNTO__ |
| if (userCommandBufferLinkMapped) |
| { |
| gcmkONERROR(gckOS_UnmapUserPointer( |
| Command->os, |
| userCommandBufferLink, |
| 0, |
| commandBufferLink)); |
| } |
| |
| if (userCommandBufferLogicalMapped) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| userCommandBufferLogical, |
| 0, |
| commandBufferLogical)); |
| } |
| #endif |
| |
| /* Unmap the command buffer pointer. */ |
| if (commandBufferMapped) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| CommandBuffer, |
| gcmSIZEOF(struct _gcoCMDBUF), |
| commandBufferObject |
| )); |
| } |
| |
| /* Return status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Reserve |
| ** |
| ** Reserve space in the command queue. Also acquire the command queue mutex. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctSIZE_T RequestedBytes |
| ** Number of bytes previously reserved. |
| ** |
| ** OUTPUT: |
| ** |
| ** gctPOINTER * Buffer |
| ** Pointer to a variable that will receive the address of the reserved |
| ** space. |
| ** |
| ** gctSIZE_T * BufferSize |
| ** Pointer to a variable that will receive the number of bytes |
| ** available in the command queue. |
| */ |
| gceSTATUS |
| gckCOMMAND_Reserve( |
| IN gckCOMMAND Command, |
| IN gctUINT32 RequestedBytes, |
| OUT gctPOINTER * Buffer, |
| OUT gctUINT32 * BufferSize |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32 bytes; |
| gctUINT32 requiredBytes; |
| gctUINT32 requestedAligned; |
| |
| gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Compute aligned number of reuested bytes. */ |
| requestedAligned = gcmALIGN(RequestedBytes, Command->alignment); |
| |
| /* Another WAIT/LINK command sequence will have to be appended after |
| the requested area being reserved. Compute the number of bytes |
| required for WAIT/LINK at the location after the reserved area. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| Command->kernel->hardware, |
| gcvNULL, |
| Command->offset + requestedAligned, |
| &requiredBytes, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| /* Compute total number of bytes required. */ |
| requiredBytes += requestedAligned; |
| |
| /* Compute number of bytes available in command queue. */ |
| bytes = Command->pageSize - Command->offset; |
| |
| /* Is there enough space in the current command queue? */ |
| if (bytes < requiredBytes) |
| { |
| /* Create a new command queue. */ |
| gcmkONERROR(_NewQueue(Command)); |
| |
| /* Recompute the number of bytes in the new kernel command queue. */ |
| bytes = Command->pageSize - Command->offset; |
| |
| /* Still not enough space? */ |
| if (bytes < requiredBytes) |
| { |
| /* Rare case, not enough room in command queue. */ |
| gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); |
| } |
| } |
| |
| /* Return pointer to empty slot command queue. */ |
| *Buffer = (gctUINT8 *) Command->logical + Command->offset; |
| |
| /* Return number of bytes left in command queue. */ |
| *BufferSize = bytes; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Buffer=0x%x *BufferSize=%lu", *Buffer, *BufferSize); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Execute |
| ** |
| ** Execute a previously reserved command queue by appending a WAIT/LINK command |
| ** sequence after it and modifying the last WAIT into a LINK command. The |
| ** command FIFO mutex will be released whether this function succeeds or not. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctSIZE_T RequestedBytes |
| ** Number of bytes previously reserved. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Execute( |
| IN gckCOMMAND Command, |
| IN gctUINT32 RequestedBytes |
| ) |
| { |
| gceSTATUS status; |
| |
| gctUINT32 waitLinkPhysical; |
| gctUINT8_PTR waitLinkLogical; |
| gctUINT32 waitLinkOffset; |
| gctUINT32 waitLinkBytes; |
| |
| gctUINT32 waitPhysical; |
| gctPOINTER waitLogical; |
| gctUINT32 waitOffset; |
| gctUINT32 waitBytes; |
| |
| gctUINT32 linkLow, linkHigh; |
| |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| gctPHYS_ADDR execPhysical; |
| #endif |
| gctPOINTER execLogical; |
| gctUINT32 execAddress; |
| gctUINT32 execBytes; |
| |
| gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Compute offset for WAIT/LINK. */ |
| waitLinkOffset = Command->offset + RequestedBytes; |
| |
| /* Compute number of bytes left in command queue. */ |
| waitLinkBytes = Command->pageSize - waitLinkOffset; |
| |
| /* Compute the location if WAIT/LINK command sequence. */ |
| waitLinkPhysical = Command->physical + waitLinkOffset; |
| waitLinkLogical = (gctUINT8_PTR) Command->logical + waitLinkOffset; |
| |
| /* Append WAIT/LINK in command queue. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| Command->kernel->hardware, |
| waitLinkLogical, |
| waitLinkOffset, |
| &waitLinkBytes, |
| &waitOffset, |
| &waitBytes |
| )); |
| |
| /* Compute the location if WAIT command. */ |
| waitPhysical = waitLinkPhysical + waitOffset; |
| waitLogical = waitLinkLogical + waitOffset; |
| |
| /* Determine the location to jump to for the command buffer being |
| ** scheduled. */ |
| if (Command->newQueue) |
| { |
| /* New command queue, jump to the beginning of it. */ |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| execPhysical = Command->physical; |
| #endif |
| execLogical = Command->logical; |
| execAddress = Command->address; |
| execBytes = waitLinkOffset + waitLinkBytes; |
| } |
| else |
| { |
| /* Still within the preexisting command queue, jump directly to the |
| reserved area. */ |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| execPhysical = (gctUINT8 *) Command->physical + Command->offset; |
| #endif |
| execLogical = (gctUINT8 *) Command->logical + Command->offset; |
| execAddress = Command->address + Command->offset; |
| execBytes = RequestedBytes + waitLinkBytes; |
| } |
| |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| /* Flush the cache. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| Command->kernelProcessID, |
| gcvNULL, |
| (gctUINT32)execPhysical, |
| execLogical, |
| execBytes |
| )); |
| #endif |
| |
| /* Convert the last WAIT into a LINK. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| Command->kernel->hardware, |
| Command->waitLogical, |
| execAddress, |
| execBytes, |
| &Command->waitSize, |
| &linkLow, |
| &linkHigh |
| )); |
| |
| #if gcdLINK_QUEUE_SIZE |
| if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) |
| { |
| gckLINKQUEUE_Enqueue( |
| &Command->kernel->hardware->linkQueue, |
| execAddress, |
| execAddress + execBytes, |
| linkLow, |
| linkHigh |
| ); |
| } |
| #endif |
| |
| #if gcdNONPAGED_MEMORY_CACHEABLE |
| /* Flush the cache. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| Command->kernelProcessID, |
| gcvNULL, |
| (gctUINT32)Command->waitPhysical, |
| Command->waitLogical, |
| Command->waitSize |
| )); |
| #endif |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| Command->waitLogical, |
| Command->waitSize, |
| gceDUMP_BUFFER_LINK, |
| gcvFALSE |
| ); |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| execLogical, |
| execBytes, |
| gceDUMP_BUFFER_KERNEL, |
| gcvFALSE |
| ); |
| |
| /* Update the pointer to the last WAIT. */ |
| Command->waitPhysical = waitPhysical; |
| Command->waitLogical = waitLogical; |
| Command->waitSize = waitBytes; |
| |
| /* Update the command queue. */ |
| Command->offset += RequestedBytes + waitLinkBytes; |
| Command->newQueue = gcvFALSE; |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail( |
| Command->kernel->hardware, Command->logical, Command->offset |
| )); |
| |
| #if gcdDUMP_COMMAND |
| gcmkPRINT("@[kernel.execute]"); |
| #endif |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Stall |
| ** |
| ** The calling thread will be suspended until the command queue has been |
| ** completed. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctBOOL FromPower |
| ** Determines whether the call originates from inside the power |
| ** management or not. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| #if gcdMULTI_GPU |
| gceSTATUS |
| gckCOMMAND_Stall( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromPower, |
| IN gceCORE_3D_MASK ChipEnable |
| ) |
| #else |
| gceSTATUS |
| gckCOMMAND_Stall( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromPower |
| ) |
| #endif |
| { |
| #if gcdNULL_DRIVER |
| /* Do nothing with infinite hardware. */ |
| return gcvSTATUS_OK; |
| #else |
| gckOS os; |
| gckHARDWARE hardware; |
| gckEVENT eventObject; |
| gceSTATUS status; |
| gctSIGNAL signal = gcvNULL; |
| gctUINT timer = 0; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Extract the gckOS object pointer. */ |
| os = Command->os; |
| gcmkVERIFY_OBJECT(os, gcvOBJ_OS); |
| |
| /* Extract the gckHARDWARE object pointer. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| /* Extract the gckEVENT object pointer. */ |
| eventObject = Command->kernel->eventObj; |
| gcmkVERIFY_OBJECT(eventObject, gcvOBJ_EVENT); |
| |
| /* Allocate the signal. */ |
| gcmkONERROR(gckOS_CreateSignal(os, gcvTRUE, &signal)); |
| |
| /* Append the EVENT command to trigger the signal. */ |
| gcmkONERROR(gckEVENT_Signal(eventObject, signal, gcvKERNEL_PIXEL)); |
| |
| /* Submit the event queue. */ |
| #if gcdMULTI_GPU |
| gcmkONERROR(gckEVENT_Submit(eventObject, gcvTRUE, FromPower, ChipEnable)); |
| #else |
| gcmkONERROR(gckEVENT_Submit(eventObject, gcvTRUE, FromPower)); |
| #endif |
| |
| #if gcdDUMP_COMMAND |
| gcmkPRINT("@[kernel.stall]"); |
| #endif |
| |
| if (status == gcvSTATUS_CHIP_NOT_READY) |
| { |
| /* Error. */ |
| goto OnError; |
| } |
| |
| do |
| { |
| /* Wait for the signal. */ |
| status = gckOS_WaitSignal(os, signal, gcdGPU_ADVANCETIMER); |
| |
| if (status == gcvSTATUS_TIMEOUT) |
| { |
| #if gcmIS_DEBUG(gcdDEBUG_CODE) |
| gctUINT32 idle; |
| |
| /* Read idle register. */ |
| gcmkVERIFY_OK(gckHARDWARE_GetIdle( |
| hardware, gcvFALSE, &idle |
| )); |
| |
| gcmkTRACE( |
| gcvLEVEL_ERROR, |
| "%s(%d): idle=%08x", |
| __FUNCTION__, __LINE__, idle |
| ); |
| |
| gcmkVERIFY_OK(gckOS_MemoryBarrier(os, gcvNULL)); |
| #endif |
| |
| /* Advance timer. */ |
| timer += gcdGPU_ADVANCETIMER; |
| } |
| else if (status == gcvSTATUS_INTERRUPTED) |
| { |
| gcmkONERROR(gcvSTATUS_INTERRUPTED); |
| } |
| |
| } |
| while (gcmIS_ERROR(status)); |
| |
| /* Bail out on timeout. */ |
| if (gcmIS_ERROR(status)) |
| { |
| /* Broadcast the stuck GPU. */ |
| gcmkONERROR(gckOS_Broadcast( |
| os, hardware, gcvBROADCAST_GPU_STUCK |
| )); |
| } |
| |
| /* Delete the signal. */ |
| gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (signal != gcvNULL) |
| { |
| /* Free the signal. */ |
| gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| #endif |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Attach |
| ** |
| ** Attach user process. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** gctUINT32 ProcessID |
| ** Current process ID. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckCONTEXT * Context |
| ** Pointer to a variable that will receive a pointer to a new |
| ** gckCONTEXT object. |
| ** |
| ** gctSIZE_T * StateCount |
| ** Pointer to a variable that will receive the number of states |
| ** in the context buffer. |
| */ |
| #if (gcdENABLE_3D || gcdENABLE_2D) |
| gceSTATUS |
| gckCOMMAND_Attach( |
| IN gckCOMMAND Command, |
| OUT gckCONTEXT * Context, |
| OUT gctSIZE_T * MaxState, |
| OUT gctUINT32 * NumStates, |
| IN gctUINT32 ProcessID |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Acquire the context switching mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, Command->mutexContext, gcvINFINITE |
| )); |
| acquired = gcvTRUE; |
| |
| /* Construct a gckCONTEXT object. */ |
| gcmkONERROR(gckCONTEXT_Construct( |
| Command->os, |
| Command->kernel->hardware, |
| ProcessID, |
| Context |
| )); |
| |
| /* Return the number of states in the context. */ |
| * MaxState = (* Context)->maxState; |
| * NumStates = (* Context)->numStates; |
| |
| /* Release the context switching mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Context=0x%x", *Context); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Release mutex. */ |
| if (acquired) |
| { |
| /* Release the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| #endif |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Detach |
| ** |
| ** Detach user process. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** gckCONTEXT Context |
| ** Pointer to a gckCONTEXT object to be destroyed. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Detach( |
| IN gckCOMMAND Command, |
| IN gckCONTEXT Context |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=0x%x Context=0x%x", Command, Context); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Acquire the context switching mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, Command->mutexContext, gcvINFINITE |
| )); |
| acquired = gcvTRUE; |
| |
| /* Construct a gckCONTEXT object. */ |
| gcmkONERROR(gckCONTEXT_Destroy(Context)); |
| |
| if (Command->currContext == Context) |
| { |
| /* Detach from gckCOMMAND object if the destoryed context is current context. */ |
| Command->currContext = gcvNULL; |
| } |
| |
| /* Release the context switching mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Release mutex. */ |
| if (acquired) |
| { |
| /* Release the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_DumpExecutingBuffer |
| ** |
| ** Dump the command buffer which GPU is executing. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_DumpExecutingBuffer( |
| IN gckCOMMAND Command |
| ) |
| { |
| gceSTATUS status; |
| gckVIRTUAL_COMMAND_BUFFER_PTR buffer = gcvNULL; |
| gctUINT32 gpuAddress; |
| gctSIZE_T pageCount; |
| gctPOINTER entry; |
| gckOS os = Command->os; |
| gckKERNEL kernel = Command->kernel; |
| gctUINT32 i; |
| gctUINT32 dumpRear; |
| gckLINKQUEUE queue = &kernel->hardware->linkQueue; |
| gctSIZE_T bytes; |
| gckLINKDATA linkData; |
| gctUINT32 offset; |
| gctPOINTER entryDump; |
| gctUINT32 pid; |
| gctUINT8 processName[24] = {0}; |
| |
| gcmkPRINT("**************************\n"); |
| gcmkPRINT("**** COMMAND BUF DUMP ****\n"); |
| gcmkPRINT("**************************\n"); |
| |
| gcmkPRINT(" Submitted commit stamp = %lld", Command->commitStamp - 1); |
| gcmkPRINT(" Executed commit stamp = %lld", *(gctUINT64_PTR)Command->fence->logical); |
| |
| gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); |
| |
| gcmkPRINT("DMA Address 0x%08X, memory around:", gpuAddress); |
| |
| /* Search and dump memory around DMA address. */ |
| if (kernel->virtualCommandBuffer) |
| { |
| status = gckKERNEL_QueryGPUAddress(kernel, gpuAddress, &buffer); |
| } |
| else |
| { |
| status = gcvSTATUS_OK; |
| } |
| |
| if (gcmIS_SUCCESS(status)) |
| { |
| if (kernel->virtualCommandBuffer) |
| { |
| gcmkVERIFY_OK(gckOS_CreateKernelVirtualMapping( |
| os, buffer->physical, buffer->bytes, &entry, &pageCount)); |
| |
| offset = gpuAddress - buffer->gpuAddress; |
| |
| entryDump = entry; |
| |
| /* Dump one pages. */ |
| bytes = 4096; |
| |
| /* Align to page. */ |
| offset &= 0xfffff000; |
| |
| /* Kernel address of page where stall point stay. */ |
| entryDump = (gctUINT8_PTR)entryDump + offset; |
| |
| /* Align to page. */ |
| gpuAddress &= 0xfffff000; |
| } |
| else |
| { |
| gcmkVERIFY_OK(gckOS_MapPhysical(os, gpuAddress, 4096, &entry)); |
| |
| /* Align to page start. */ |
| entryDump = (gctPOINTER)((gctUINTPTR_T)entry & ~0xFFF); |
| gpuAddress = gpuAddress & ~0xFFF; |
| bytes = 4096; |
| } |
| |
| gcmkPRINT("User Command Buffer:\n"); |
| _DumpBuffer(entryDump, gpuAddress, bytes); |
| |
| if (kernel->virtualCommandBuffer) |
| { |
| gcmkVERIFY_OK(gckOS_DestroyKernelVirtualMapping( |
| os, buffer->physical, buffer->bytes, entry)); |
| } |
| else |
| { |
| gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, 4096)); |
| } |
| } |
| else |
| { |
| _DumpKernelCommandBuffer(Command); |
| } |
| |
| /* Dump link queue. */ |
| if (queue->count) |
| { |
| gcmkPRINT("Dump Level is %d, dump %d valid record in link queue:", |
| Command->kernel->stuckDump, queue->count); |
| |
| dumpRear = queue->count; |
| |
| for (i = 0; i < dumpRear; i++) |
| { |
| gckLINKQUEUE_GetData(queue, i, &linkData); |
| |
| /* Get gpu address of this command buffer. */ |
| gpuAddress = linkData->start; |
| bytes = linkData->end - gpuAddress; |
| |
| pid = linkData->pid; |
| |
| gckOS_GetProcessNameByPid(pid, 16, processName); |
| |
| if (kernel->virtualCommandBuffer) |
| { |
| buffer = gcvNULL; |
| |
| /* Get the whole buffer. */ |
| status = gckKERNEL_QueryGPUAddress(kernel, gpuAddress, &buffer); |
| |
| if (gcmIS_ERROR(status)) |
| { |
| /* Get kernel address of kernel command buffer. */ |
| status = gckCOMMAND_AddressInKernelCommandBuffer( |
| kernel->command, gpuAddress, &entry); |
| |
| if (gcmIS_ERROR(status)) |
| { |
| status = gckHARDWARE_AddressInHardwareFuncions( |
| kernel->hardware, gpuAddress, &entry); |
| |
| if (gcmIS_ERROR(status)) |
| { |
| gcmkPRINT("Buffer [%08X - %08X] not found, may be freed", |
| linkData->start, |
| linkData->end); |
| continue; |
| } |
| } |
| |
| offset = 0; |
| gcmkPRINT("Kernel Command Buffer: %08X, %08X", linkData->linkLow, linkData->linkHigh); |
| } |
| else |
| { |
| /* Get kernel logical for dump. */ |
| if (buffer->kernelLogical) |
| { |
| /* Get kernel logical directly if it is a context buffer. */ |
| entry = buffer->kernelLogical; |
| gcmkPRINT("Context Buffer: %08X, %08X PID:%d %s", |
| linkData->linkLow, linkData->linkHigh, linkData->pid, processName); |
| } |
| else |
| { |
| /* Make it accessiable by kernel if it is a user command buffer. */ |
| gcmkVERIFY_OK( |
| gckOS_CreateKernelVirtualMapping(os, |
| buffer->physical, |
| buffer->bytes, |
| &entry, |
| &pageCount)); |
| gcmkPRINT("User Command Buffer: %08X, %08X PID:%d %s", |
| linkData->linkLow, linkData->linkHigh, linkData->pid, processName); |
| } |
| |
| offset = gpuAddress - buffer->gpuAddress; |
| } |
| |
| /* Dump from the entry. */ |
| _DumpBuffer((gctUINT8_PTR)entry + offset, gpuAddress, bytes); |
| |
| /* Release kernel logical address if neccessary. */ |
| if (buffer && !buffer->kernelLogical) |
| { |
| gcmkVERIFY_OK( |
| gckOS_DestroyKernelVirtualMapping(os, |
| buffer->physical, |
| buffer->bytes, |
| entry)); |
| } |
| } |
| else |
| { |
| gcmkVERIFY_OK(gckOS_MapPhysical(os, gpuAddress, bytes, &entry)); |
| |
| gcmkPRINT("Command Buffer: %08X, %08X PID:%d %s", |
| linkData->linkLow, linkData->linkHigh, linkData->pid, processName); |
| |
| _DumpBuffer((gctUINT8_PTR)entry, gpuAddress, bytes); |
| |
| gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, bytes)); |
| } |
| } |
| } |
| |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckCOMMAND_AddressInKernelCommandBuffer( |
| IN gckCOMMAND Command, |
| IN gctUINT32 Address, |
| OUT gctPOINTER * Pointer |
| ) |
| { |
| gctINT i; |
| |
| for (i = 0; i < gcdCOMMAND_QUEUES; i++) |
| { |
| if ((Address >= Command->queues[i].address) |
| && (Address < (Command->queues[i].address + Command->pageSize)) |
| ) |
| { |
| *Pointer = (gctUINT8_PTR)Command->queues[i].logical |
| + (Address - Command->queues[i].address) |
| ; |
| |
| return gcvSTATUS_OK; |
| } |
| } |
| |
| return gcvSTATUS_NOT_FOUND; |
| } |