Google Git
Sign in
nest-open-source / nest-learning-thermostat / 6.0 / linux-imx-4.9.88 / f8cc81dfff00c26ff85530a8a7c19533694b47b8 / . / drivers / mxc / gpu-viv / hal / kernel / gc_hal_kernel_command.c
blob: 7a6511828b1856c4d5e2a089eaa4d4bc4305d794 [file] [log] [blame]
/****************************************************************************
*
* The MIT License (MIT)
*
* Copyright (c) 2014 - 2018 Vivante Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
*****************************************************************************
*
* The GPL License (GPL)
*
* Copyright (C) 2014 - 2018 Vivante Corporation
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*****************************************************************************
*
* Note: This software is released under dual MIT and GPL licenses. A
* recipient may use this file under the terms of either the MIT license or
* GPL License. If you wish to use only one license not the other, you can
* indicate your decision by deleting one of the above license notices in your
* version of this file.
*
*****************************************************************************/
#include "gc_hal_kernel_precomp.h"
#include "gc_hal_kernel_context.h"
#define _GC_OBJ_ZONE gcvZONE_COMMAND
/******************************************************************************\
********************************* Support Code *********************************
\******************************************************************************/
/*******************************************************************************
**
** _NewQueue
**
** Allocate a new command queue.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to an gckCOMMAND object.
**
** gctBOOL Stalled
** Indicate if hardware is stalled already.
**
** OUTPUT:
**
** gckCOMMAND Command
** gckCOMMAND object has been updated with a new command queue.
*/
static gceSTATUS
_NewQueue(
IN OUT gckCOMMAND Command,
IN gctBOOL Stalled
)
{
gceSTATUS status;
gctINT currentIndex, newIndex;
gctPHYS_ADDR_T physical;
gcmkHEADER_ARG("Command=0x%x", Command);
/* Switch to the next command buffer. */
currentIndex = Command->index;
newIndex = (currentIndex + 1) % gcdCOMMAND_QUEUES;
/* Wait for availability. */
#if gcdDUMP_COMMAND
gcmkPRINT("@[kernel.waitsignal]");
#endif
gcmkONERROR(gckOS_WaitSignal(
Command->os,
Command->queues[newIndex].signal,
gcvFALSE,
gcvINFINITE
));
#if gcmIS_DEBUG(gcdDEBUG_TRACE)
if (newIndex < currentIndex)
{
Command->wrapCount += 1;
gcmkTRACE_ZONE_N(
gcvLEVEL_INFO, gcvZONE_COMMAND,
2 * 4,
"%s(%d): queue array wrapped around.\n",
__FUNCTION__, __LINE__
);
}
gcmkTRACE_ZONE_N(
gcvLEVEL_INFO, gcvZONE_COMMAND,
3 * 4,
"%s(%d): total queue wrap arounds %d.\n",
__FUNCTION__, __LINE__, Command->wrapCount
);
gcmkTRACE_ZONE_N(
gcvLEVEL_INFO, gcvZONE_COMMAND,
3 * 4,
"%s(%d): switched to queue %d.\n",
__FUNCTION__, __LINE__, newIndex
);
#endif
/* Update gckCOMMAND object with new command queue. */
Command->index = newIndex;
Command->newQueue = gcvTRUE;
Command->virtualMemory = Command->queues[newIndex].physical;
Command->logical = Command->queues[newIndex].logical;
Command->address = Command->queues[newIndex].address;
Command->offset = 0;
gcmkONERROR(gckOS_GetPhysicalAddress(
Command->os,
Command->logical,
&physical
));
gcmkSAFECASTPHYSADDRT(Command->physical, physical);
if (currentIndex != -1)
{
if (Stalled)
{
gckOS_Signal(
Command->os,
Command->queues[currentIndex].signal,
gcvTRUE
);
}
else
{
/* Mark the command queue as available. */
gcmkONERROR(gckEVENT_Signal(
Command->kernel->eventObj,
Command->queues[currentIndex].signal,
gcvKERNEL_COMMAND
));
}
}
/* Success. */
gcmkFOOTER_ARG("Command->index=%d", Command->index);
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
static gceSTATUS
_IncrementCommitAtom(
IN gckCOMMAND Command,
IN gctBOOL Increment
)
{
gceSTATUS status;
gckHARDWARE hardware;
gctINT32 atomValue;
gctBOOL powerAcquired = gcvFALSE;
gcmkHEADER_ARG("Command=0x%x", Command);
/* Extract the gckHARDWARE and gckEVENT objects. */
hardware = Command->kernel->hardware;
gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
/* Grab the power mutex. */
gcmkONERROR(gckOS_AcquireMutex(
Command->os, hardware->powerMutex, gcvINFINITE
));
powerAcquired = gcvTRUE;
/* Increment the commit atom. */
if (Increment)
{
gcmkONERROR(gckOS_AtomIncrement(
Command->os, Command->atomCommit, &atomValue
));
}
else
{
gcmkONERROR(gckOS_AtomDecrement(
Command->os, Command->atomCommit, &atomValue
));
}
/* Release the power mutex. */
gcmkONERROR(gckOS_ReleaseMutex(
Command->os, hardware->powerMutex
));
powerAcquired = gcvFALSE;
/* Success. */
gcmkFOOTER();
return gcvSTATUS_OK;
OnError:
if (powerAcquired)
{
/* Release the power mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(
Command->os, hardware->powerMutex
));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
#if gcdSECURE_USER
static gceSTATUS
_ProcessHints(
IN gckCOMMAND Command,
IN gctUINT32 ProcessID,
IN gcoCMDBUF CommandBuffer
)
{
gceSTATUS status = gcvSTATUS_OK;
gckKERNEL kernel;
gctBOOL needCopy = gcvFALSE;
gcskSECURE_CACHE_PTR cache;
gctUINT8_PTR commandBufferLogical;
gctUINT8_PTR hintedData;
gctUINT32_PTR hintArray;
gctUINT i, hintCount;
gcmkHEADER_ARG(
"Command=0x%08X ProcessID=%d CommandBuffer=0x%08X",
Command, ProcessID, CommandBuffer
);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
/* Reset state array pointer. */
hintArray = gcvNULL;
/* Get the kernel object. */
kernel = Command->kernel;
/* Get the cache form the database. */
gcmkONERROR(gckKERNEL_GetProcessDBCache(kernel, ProcessID, &cache));
/* Determine the start of the command buffer. */
commandBufferLogical
= (gctUINT8_PTR) CommandBuffer->logical
+ CommandBuffer->startOffset;
/* Determine the number of records in the state array. */
hintCount = CommandBuffer->hintArrayTail - CommandBuffer->hintArray;
/* Check wehther we need to copy the structures or not. */
gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy));
/* Get access to the state array. */
if (needCopy)
{
gctUINT copySize;
if (Command->hintArrayAllocated &&
(Command->hintArraySize < CommandBuffer->hintArraySize))
{
gcmkONERROR(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray)));
Command->hintArraySize = gcvFALSE;
}
if (!Command->hintArrayAllocated)
{
gctPOINTER pointer = gcvNULL;
gcmkONERROR(gckOS_Allocate(
Command->os,
CommandBuffer->hintArraySize,
&pointer
));
Command->hintArray = gcmPTR_TO_UINT64(pointer);
Command->hintArrayAllocated = gcvTRUE;
Command->hintArraySize = CommandBuffer->hintArraySize;
}
hintArray = gcmUINT64_TO_PTR(Command->hintArray);
copySize = hintCount * gcmSIZEOF(gctUINT32);
gcmkONERROR(gckOS_CopyFromUserData(
Command->os,
hintArray,
gcmUINT64_TO_PTR(CommandBuffer->hintArray),
copySize
));
}
else
{
gctPOINTER pointer = gcvNULL;
gcmkONERROR(gckOS_MapUserPointer(
Command->os,
gcmUINT64_TO_PTR(CommandBuffer->hintArray),
CommandBuffer->hintArraySize,
&pointer
));
hintArray = pointer;
}
/* Scan through the buffer. */
for (i = 0; i < hintCount; i += 1)
{
/* Determine the location of the hinted data. */
hintedData = commandBufferLogical + hintArray[i];
/* Map handle into physical address. */
gcmkONERROR(gckKERNEL_MapLogicalToPhysical(
kernel, cache, (gctPOINTER) hintedData
));
}
OnError:
/* Get access to the state array. */
if (!needCopy && (hintArray != gcvNULL))
{
gcmkVERIFY_OK(gckOS_UnmapUserPointer(
Command->os,
gcmUINT64_TO_PTR(CommandBuffer->hintArray),
CommandBuffer->hintArraySize,
hintArray
));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
#endif
#if !gcdNULL_DRIVER
static gceSTATUS
_FlushMMU(
IN gckCOMMAND Command
)
{
#if gcdSECURITY
return gcvSTATUS_OK;
#else
gceSTATUS status;
gctUINT32 oldValue;
gckHARDWARE hardware = Command->kernel->hardware;
gctBOOL pause = gcvFALSE;
gctUINT8_PTR pointer;
gctUINT32 address;
gctUINT32 eventBytes;
gctUINT32 endBytes;
gctUINT32 bufferSize;
gctUINT32 executeBytes;
gctUINT32 waitLinkBytes;
gcmkONERROR(gckOS_AtomicExchange(Command->os,
hardware->pageTableDirty[gcvENGINE_RENDER],
0,
&oldValue));
if (oldValue)
{
/* Page Table is upated, flush mmu before commit. */
gcmkONERROR(gckHARDWARE_FlushMMU(hardware));
if ((oldValue & gcvPAGE_TABLE_DIRTY_BIT_FE)
&& (!hardware->stallFEPrefetch)
)
{
pause = gcvTRUE;
}
}
if (pause)
{
/* Query size. */
gcmkONERROR(gckHARDWARE_Event(hardware, gcvNULL, 0, gcvKERNEL_PIXEL, &eventBytes));
gcmkONERROR(gckHARDWARE_End(hardware, gcvNULL, ~0U, &endBytes));
executeBytes = eventBytes + endBytes;
gcmkONERROR(gckHARDWARE_WaitLink(
hardware,
gcvNULL,
~0U,
Command->offset + executeBytes,
&waitLinkBytes,
gcvNULL,
gcvNULL
));
/* Reserve space. */
gcmkONERROR(gckCOMMAND_Reserve(
Command,
executeBytes,
(gctPOINTER *)&pointer,
&bufferSize
));
/* Pointer to reserved address. */
address = Command->address + Command->offset;
/* Append EVENT(29). */
gcmkONERROR(gckHARDWARE_Event(
hardware,
pointer,
29,
gcvKERNEL_PIXEL,
&eventBytes
));
/* Append END. */
pointer += eventBytes;
address += eventBytes;
gcmkONERROR(gckHARDWARE_End(hardware, pointer, address, &endBytes));
#if USE_KERNEL_VIRTUAL_BUFFERS
if (hardware->kernel->virtualCommandBuffer)
{
gcmkONERROR(gckKERNEL_GetGPUAddress(
hardware->kernel,
pointer,
gcvFALSE,
Command->virtualMemory,
&hardware->lastEnd
));
}
#endif
gcmkONERROR(gckCOMMAND_Execute(Command, executeBytes));
}
return gcvSTATUS_OK;
OnError:
return status;
#endif
}
static gceSTATUS
_DummyDraw(
IN gckCOMMAND Command
)
{
#if gcdSECURITY
return gcvSTATUS_OK;
#else
gceSTATUS status;
gckHARDWARE hardware = Command->kernel->hardware;
gctUINT8_PTR pointer;
gctUINT32 bufferSize;
gctUINT32 dummyDrawBytes;
gceDUMMY_DRAW_TYPE dummyDrawType = gcvDUMMY_DRAW_INVALID;
if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FE_NEED_DUMMYDRAW))
{
dummyDrawType = gcvDUMMY_DRAW_GC400;
}
if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC_DEFER_FILL_FIX) &&
gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC))
{
dummyDrawType = gcvDUMMY_DRAW_V60;
}
if (dummyDrawType != gcvDUMMY_DRAW_INVALID)
{
gckHARDWARE_DummyDraw(hardware, gcvNULL, Command->queues[0].address, dummyDrawType, &dummyDrawBytes);
/* Reserve space. */
gcmkONERROR(gckCOMMAND_Reserve(
Command,
dummyDrawBytes,
(gctPOINTER *)&pointer,
&bufferSize
));
gckHARDWARE_DummyDraw(hardware, pointer, Command->queues[0].address, dummyDrawType, &dummyDrawBytes);
gcmkONERROR(gckCOMMAND_Execute(Command, dummyDrawBytes));
}
return gcvSTATUS_OK;
OnError:
return status;
#endif
}
#endif
static void
_DumpBuffer(
IN gctPOINTER Buffer,
IN gctUINT32 GpuAddress,
IN gctSIZE_T Size
)
{
gctSIZE_T i, line, left;
gctUINT32_PTR data = Buffer;
line = Size / 32;
left = Size % 32;
for (i = 0; i < line; i++)
{
gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X %08X",
GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]);
data += 8;
GpuAddress += 8 * 4;
}
switch(left)
{
case 28:
gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X",
GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6]);
break;
case 24:
gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X",
GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5]);
break;
case 20:
gcmkPRINT("%08X : %08X %08X %08X %08X %08X",
GpuAddress, data[0], data[1], data[2], data[3], data[4]);
break;
case 16:
gcmkPRINT("%08X : %08X %08X %08X %08X",
GpuAddress, data[0], data[1], data[2], data[3]);
break;
case 12:
gcmkPRINT("%08X : %08X %08X %08X",
GpuAddress, data[0], data[1], data[2]);
break;
case 8:
gcmkPRINT("%08X : %08X %08X",
GpuAddress, data[0], data[1]);
break;
case 4:
gcmkPRINT("%08X : %08X",
GpuAddress, data[0]);
break;
default:
break;
}
}
static void
_DumpKernelCommandBuffer(
IN gckCOMMAND Command
)
{
gctINT i;
gctUINT64 physical = 0;
gctUINT32 address;
gctPOINTER entry = gcvNULL;
for (i = 0; i < gcdCOMMAND_QUEUES; i++)
{
entry = Command->queues[i].logical;
gckOS_GetPhysicalAddress(Command->os, entry, &physical);
gcmkPRINT("Kernel command buffer %d\n", i);
gcmkSAFECASTPHYSADDRT(address, physical);
_DumpBuffer(entry, address, Command->pageSize);
}
}
#if !gcdNULL_DRIVER
static gceSTATUS
_HandlePatchList(
IN gckCOMMAND Command,
IN gcoCMDBUF CommandBuffer,
IN gctBOOL NeedCopy,
OUT gctUINT64 *AsyncCommandStamp
)
{
gceSTATUS status;
gcsPATCH_LIST * uList;
gcsPATCH_LIST * previous;
gcsPATCH_LIST * kList;
gctUINT64 asyncStamp = 0;
gcmkHEADER_ARG(
"Command=0x%x CommandBuffer=0x%x NeedCopy=%d",
Command, CommandBuffer, NeedCopy
);
uList = gcmUINT64_TO_PTR(CommandBuffer->patchHead);
while (uList)
{
gctUINT i;
kList = gcvNULL;
previous = uList;
gcmkONERROR(gckKERNEL_OpenUserData(
Command->kernel,
NeedCopy,
Command->kList,
uList,
gcmSIZEOF(gcsPATCH_LIST),
(gctPOINTER *)&kList
));
for (i = 0; i < kList->count; i++)
{
gctUINT64 stamp = 0;
gcsPATCH * patch = &kList->patch[i];
/* Touch video memory node. */
gcmkVERIFY_OK(gckVIDMEM_SetCommitStamp(Command->kernel, gcvENGINE_RENDER, patch->handle, Command->commitStamp));
/* Get stamp touched async command buffer. */
gcmkVERIFY_OK(gckVIDMEM_GetCommitStamp(Command->kernel, gcvENGINE_BLT, patch->handle, &stamp));
/* Find latest one. */
asyncStamp = gcmMAX(asyncStamp, stamp);
}
uList = kList->next;
gcmkVERIFY_OK(gckKERNEL_CloseUserData(
Command->kernel,
NeedCopy,
gcvFALSE,
previous,
gcmSIZEOF(gcsPATCH_LIST),
(gctPOINTER *)&kList
));
}
if ((Command->asyncCommand != gcvNULL)
&& (*(gctUINT64 *)Command->asyncCommand->fence->logical > asyncStamp)
)
{
/* No need to wait for async command buffer. */
*AsyncCommandStamp = 0;
}
else
{
/* Need to add a fence wait. */
*AsyncCommandStamp = asyncStamp;
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (kList)
{
gcmkVERIFY_OK(gckKERNEL_CloseUserData(
Command->kernel,
NeedCopy,
gcvFALSE,
previous,
gcmSIZEOF(gcsPATCH_LIST),
(gctPOINTER *)&kList
));
}
gcmkFOOTER();
return status;
}
static gceSTATUS
_WaitForAsyncCommandStamp(
IN gckCOMMAND Command,
IN gctUINT64 Stamp
)
{
gctUINT32 bytes;
gceSTATUS status;
gctUINT32 fenceAddress;
gctUINT32 bufferSize;
gctPOINTER pointer;
gcmkHEADER_ARG("Stamp = 0x%llx", Stamp);
fenceAddress = Command->asyncCommand->fence->address;
gcmkONERROR(gckHARDWARE_WaitFence(Command->kernel->hardware,
gcvNULL,
Stamp,
fenceAddress,
&bytes
));
gcmkONERROR(gckCOMMAND_Reserve(
Command,
bytes,
&pointer,
&bufferSize
));
gcmkONERROR(gckHARDWARE_WaitFence(
Command->kernel->hardware,
pointer,
Stamp,
fenceAddress,
&bytes
));
gcmkONERROR(gckCOMMAND_Execute(Command, bytes));
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
/******************************************************************************\
**************** Helper functions for parsing gcoCMDBUF ************************
\******************************************************************************/
static void
_GetCMDBUFSize(
IN gcoCMDBUF CommandBuffer,
OUT gctUINT_PTR CommandBufferSize
)
{
*CommandBufferSize
= CommandBuffer->offset
+ CommandBuffer->reservedTail
- CommandBuffer->startOffset;
}
static void
_GetCMDBUFTail(
IN gcoCMDBUF CommandBuffer,
OUT gctUINT8_PTR * Tail
)
{
gctUINT8_PTR commandBufferLogical;
gctUINT commandBufferSize;
commandBufferLogical
= (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical)
+ CommandBuffer->startOffset;
_GetCMDBUFSize(CommandBuffer, &commandBufferSize);
*Tail
= commandBufferLogical
+ commandBufferSize
- CommandBuffer->reservedTail;
}
static void
_ParseCMDBUFTail(
IN gckHARDWARE Hardware,
IN gcoCMDBUF CommandBuffer,
OUT gctUINT8_PTR * Fence,
OUT gctUINT8_PTR * Link
)
{
gctUINT8_PTR tail;
_GetCMDBUFTail(CommandBuffer, &tail);
if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_FENCE_64BIT))
{
*Fence = tail;
*Link = tail + gcdRENDER_FENCE_LENGTH;
}
else
{
*Fence = gcvNULL;
*Link = tail;
}
}
static gceSTATUS
_GetCMDBUFEntry(
IN gckCOMMAND Command,
IN gcoCMDBUF CommandBuffer,
OUT gctUINT32_PTR EntryAddress,
OUT gctUINT32_PTR EntryBytes
)
{
gceSTATUS status;
gctUINT8_PTR commandBufferLogical;
gctUINT commandBufferSize;
gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer;
gctUINT32 commandBufferAddress;
gctUINT offset;
commandBufferLogical
= (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical)
+ CommandBuffer->startOffset;
/* Get the hardware address. */
if (Command->kernel->virtualCommandBuffer)
{
gckKERNEL kernel = Command->kernel;
virtualCommandBuffer = gcmNAME_TO_PTR(CommandBuffer->physical);
if (virtualCommandBuffer == gcvNULL)
{
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
gcmkONERROR(gckKERNEL_GetGPUAddress(
Command->kernel,
commandBufferLogical,
gcvTRUE,
virtualCommandBuffer,
&commandBufferAddress
));
}
else
{
gcmkONERROR(gckHARDWARE_ConvertLogical(
Command->kernel->hardware,
commandBufferLogical,
gcvTRUE,
&commandBufferAddress
));
}
/* Get offset. */
gcmkONERROR(gckHARDWARE_PipeSelect(
Command->kernel->hardware, gcvNULL, gcvPIPE_3D, &offset
));
_GetCMDBUFSize(CommandBuffer, &commandBufferSize);
*EntryAddress = commandBufferAddress + offset;
*EntryBytes = commandBufferSize - offset;
return gcvSTATUS_OK;
OnError:
return status;
}
/*******************************************************************************
**
** Link a list of command buffer together to make them atomic.
** Fence will be added in the last command buffer.
*/
static gceSTATUS
_ProcessUserCommandBufferList(
IN gckCOMMAND Command,
IN gcoCMDBUF CommandBufferListHead,
OUT gcoCMDBUF * CommandBufferListTail
)
{
gceSTATUS status;
gctBOOL needCopy;
struct _gcoCMDBUF _commandBufferObject;
gcoCMDBUF currentCMDBUF;
struct _gcoCMDBUF _nextCMDBUF;
gcoCMDBUF currentCMDBUFUser = CommandBufferListHead;
gckOS_QueryNeedCopy(Command->os, 0, &needCopy);
/* Open first gcoCMDBUF object as currentCMDBUF. */
gcmkONERROR(gckKERNEL_OpenUserData(
Command->kernel,
needCopy,
&_commandBufferObject,
currentCMDBUFUser,
gcmSIZEOF(struct _gcoCMDBUF),
(gctPOINTER *)&currentCMDBUF
));
/* Iterate the list. */
while (currentCMDBUF->nextCMDBUF != 0)
{
gcoCMDBUF nextCMDBUFUser;
gcoCMDBUF nextCMDBUF;
gctUINT8_PTR fenceLogical = gcvNULL;
gctUINT8_PTR linkLogical;
gctUINT32 linkBytes = 8;
gctUINT32 linkLow;
gctUINT32 linkHigh;
gctUINT32 entryAddress = 0;
gctUINT32 entryBytes = 0;
nextCMDBUFUser
= gcmUINT64_TO_PTR(currentCMDBUF->nextCMDBUF);
/* Open next gcoCMDBUF object as nextCMDBUF. */
gcmkONERROR(gckKERNEL_OpenUserData(
Command->kernel,
needCopy,
&_nextCMDBUF,
nextCMDBUFUser,
gcmSIZEOF(struct _gcoCMDBUF),
(gctPOINTER *)&nextCMDBUF
));
/* Get the start hardware address of nextCMDBUF. */
gcmkONERROR(_GetCMDBUFEntry(Command,
nextCMDBUF,
&entryAddress,
&entryBytes
));
/* Process current gcoCMDBUF object. */
_ParseCMDBUFTail(
Command->kernel->hardware,
currentCMDBUF,
&fenceLogical,
&linkLogical
);
/* Don't send fence in the middle of gcoCMDBUF list. */
if (fenceLogical != gcvNULL)
{
gctUINT i = gcdRENDER_FENCE_LENGTH / gcmSIZEOF(gctUINT32) / 2;
/* Fill NOPs in space reserved for fence. */
while (i--)
{
gctSIZE_T nopBytes = 8;
gcmkONERROR(gckHARDWARE_Nop(Command->kernel->hardware, fenceLogical, &nopBytes));
fenceLogical += nopBytes;
}
}
/* Generate a LINK from the end of current command buffer
** to the start of next command buffer. */
gcmkONERROR(gckHARDWARE_Link(
Command->kernel->hardware,
linkLogical,
entryAddress,
entryBytes,
&linkBytes,
&linkLow,
&linkHigh
));
/* Close current gcoCMDBUF object which is processed. */
gcmkVERIFY_OK(gckKERNEL_CloseUserData(
Command->kernel,
needCopy,
gcvFALSE,
currentCMDBUFUser,
gcmSIZEOF(struct _gcoCMDBUF),
(gctPOINTER *)&currentCMDBUF
));
/* Advance to next gcoCMDBUF object. */
currentCMDBUFUser = nextCMDBUFUser;
currentCMDBUF = nextCMDBUF;
}
gcmkVERIFY_OK(gckKERNEL_CloseUserData(
Command->kernel,
needCopy,
gcvFALSE,
currentCMDBUFUser,
gcmSIZEOF(struct _gcoCMDBUF),
(gctPOINTER *)&currentCMDBUF
));
/* Return the tail of the list. */
*CommandBufferListTail = currentCMDBUFUser;
return gcvSTATUS_OK;
OnError:
return status;
}
#endif
/******************************************************************************\
****************************** gckCOMMAND API Code ******************************
\******************************************************************************/
/*******************************************************************************
**
** gckCOMMAND_Construct
**
** Construct a new gckCOMMAND object.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** OUTPUT:
**
** gckCOMMAND * Command
** Pointer to a variable that will hold the pointer to the gckCOMMAND
** object.
*/
gceSTATUS
gckCOMMAND_Construct(
IN gckKERNEL Kernel,
OUT gckCOMMAND * Command
)
{
gckOS os;
gckCOMMAND command = gcvNULL;
gceSTATUS status;
gctINT i;
gctPOINTER pointer = gcvNULL;
gctSIZE_T pageSize;
gcmkHEADER_ARG("Kernel=0x%x", Kernel);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(Command != gcvNULL);
/* Extract the gckOS object. */
os = Kernel->os;
/* Allocate the gckCOMMAND structure. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckCOMMAND), &pointer));
command = pointer;
/* Reset the entire object. */
gcmkONERROR(gckOS_ZeroMemory(command, gcmSIZEOF(struct _gckCOMMAND)));
/* Initialize the gckCOMMAND object.*/
command->object.type = gcvOBJ_COMMAND;
command->kernel = Kernel;
command->os = os;
/* Get the command buffer requirements. */
gcmkONERROR(gckHARDWARE_QueryCommandBuffer(
Kernel->hardware,
gcvENGINE_RENDER,
&command->alignment,
&command->reservedHead,
gcvNULL
));
/* Create the command queue mutex. */
gcmkONERROR(gckOS_CreateMutex(os, &command->mutexQueue));
/* Create the context switching mutex. */
gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContext));
/* Create the context switching mutex. */
gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContextSeq));
/* Create the power management semaphore. */
gcmkONERROR(gckOS_CreateSemaphore(os, &command->powerSemaphore));
/* Create the commit atom. */
gcmkONERROR(gckOS_AtomConstruct(os, &command->atomCommit));
/* Get the page size from teh OS. */
gcmkONERROR(gckOS_GetPageSize(os, &pageSize));
gcmkSAFECASTSIZET(command->pageSize, pageSize);
/* Get process ID. */
gcmkONERROR(gckOS_GetProcessID(&command->kernelProcessID));
/* Set hardware to pipe 0. */
command->pipeSelect = gcvPIPE_INVALID;
/* Pre-allocate the command queues. */
for (i = 0; i < gcdCOMMAND_QUEUES; ++i)
{
#if USE_KERNEL_VIRTUAL_BUFFERS
if (Kernel->virtualCommandBuffer)
{
gcmkONERROR(gckKERNEL_AllocateVirtualCommandBuffer(
Kernel,
gcvFALSE,
&pageSize,
&command->queues[i].physical,
&command->queues[i].logical
));
gcmkONERROR(gckKERNEL_GetGPUAddress(
Kernel,
command->queues[i].logical,
gcvFALSE,
command->queues[i].physical,
&command->queues[i].address
));
}
else
#endif
{
gcmkONERROR(gckOS_AllocateNonPagedMemory(
os,
gcvFALSE,
&pageSize,
&command->queues[i].physical,
&command->queues[i].logical
));
gcmkONERROR(gckHARDWARE_ConvertLogical(
Kernel->hardware,
command->queues[i].logical,
gcvFALSE,
&command->queues[i].address
));
gcmkONERROR(gckMMU_FillFlatMapping(
Kernel->mmu, command->queues[i].address, pageSize
));
}
gcmkONERROR(gckOS_CreateSignal(
os, gcvFALSE, &command->queues[i].signal
));
gcmkONERROR(gckOS_Signal(
os, command->queues[i].signal, gcvTRUE
));
}
#if gcdRECORD_COMMAND
gcmkONERROR(gckRECORDER_Construct(os, Kernel->hardware, &command->recorder));
#endif
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsPATCH_LIST), &command->kList));
gcmkONERROR(gckFENCE_Create(
os, Kernel, &command->fence
));
/* No command queue in use yet. */
command->index = -1;
command->logical = gcvNULL;
command->newQueue = gcvFALSE;
/* Command is not yet running. */
command->running = gcvFALSE;
/* Command queue is idle. */
command->idle = gcvTRUE;
/* Commit stamp start from 1. */
command->commitStamp = 1;
/* END event signal not created. */
command->endEventSignal = gcvNULL;
command->dummyDraw = gcvTRUE;
/* Return pointer to the gckCOMMAND object. */
*Command = command;
/* Success. */
gcmkFOOTER_ARG("*Command=0x%x", *Command);
return gcvSTATUS_OK;
OnError:
/* Roll back. */
if (command != gcvNULL)
{
gcmkVERIFY_OK(gckCOMMAND_Destroy(command));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckCOMMAND_Destroy
**
** Destroy an gckCOMMAND object.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to an gckCOMMAND object to destroy.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckCOMMAND_Destroy(
IN gckCOMMAND Command
)
{
gctINT i;
gcmkHEADER_ARG("Command=0x%x", Command);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
/* Stop the command queue. */
gcmkVERIFY_OK(gckCOMMAND_Stop(Command));
for (i = 0; i < gcdCOMMAND_QUEUES; ++i)
{
if (Command->queues[i].signal)
{
gcmkVERIFY_OK(gckOS_DestroySignal(
Command->os, Command->queues[i].signal
));
}
if (Command->queues[i].logical)
{
#if USE_KERNEL_VIRTUAL_BUFFERS
if (Command->kernel->virtualCommandBuffer)
{
gcmkVERIFY_OK(gckKERNEL_DestroyVirtualCommandBuffer(
Command->kernel,
Command->pageSize,
Command->queues[i].physical,
Command->queues[i].logical
));
}
else
#endif
{
gcmkVERIFY_OK(gckOS_FreeNonPagedMemory(
Command->os,
Command->pageSize,
Command->queues[i].physical,
Command->queues[i].logical
));
}
}
}
/* END event signal. */
if (Command->endEventSignal != gcvNULL)
{
gcmkVERIFY_OK(gckOS_DestroySignal(
Command->os, Command->endEventSignal
));
}
if (Command->mutexContext)
{
/* Delete the context switching mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContext));
}
if (Command->mutexContextSeq != gcvNULL)
gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContextSeq));
if (Command->mutexQueue)
{
/* Delete the command queue mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexQueue));
}
if (Command->powerSemaphore)
{
/* Destroy the power management semaphore. */
gcmkVERIFY_OK(gckOS_DestroySemaphore(Command->os, Command->powerSemaphore));
}
if (Command->atomCommit)
{
/* Destroy the commit atom. */
gcmkVERIFY_OK(gckOS_AtomDestroy(Command->os, Command->atomCommit));
}
#if gcdSECURE_USER
/* Free state array. */
if (Command->hintArrayAllocated)
{
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray)));
Command->hintArrayAllocated = gcvFALSE;
}
#endif
#if gcdRECORD_COMMAND
gckRECORDER_Destory(Command->os, Command->recorder);
#endif
if (Command->stateMap)
{
gcmkOS_SAFE_FREE(Command->os, Command->stateMap);
}
if (Command->kList)
{
gcmkOS_SAFE_FREE(Command->os, Command->kList);
}
if (Command->fence)
{
gcmkVERIFY_OK(gckFENCE_Destory(Command->os, Command->fence));
}
/* Mark object as unknown. */
Command->object.type = gcvOBJ_UNKNOWN;
/* Free the gckCOMMAND object. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, Command));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckCOMMAND_EnterCommit
**
** Acquire command queue synchronization objects.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to an gckCOMMAND object to destroy.
**
** gctBOOL FromPower
** Determines whether the call originates from inside the power
** management or not.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckCOMMAND_EnterCommit(
IN gckCOMMAND Command,
IN gctBOOL FromPower
)
{
gceSTATUS status;
gckHARDWARE hardware;
gctBOOL atomIncremented = gcvFALSE;
gctBOOL semaAcquired = gcvFALSE;
gcmkHEADER_ARG("Command=0x%x", Command);
/* Extract the gckHARDWARE and gckEVENT objects. */
hardware = Command->kernel->hardware;
gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
if (!FromPower)
{
/* Increment COMMIT atom to let power management know that a commit is
** in progress. */
gcmkONERROR(_IncrementCommitAtom(Command, gcvTRUE));
atomIncremented = gcvTRUE;
/* Notify the system the GPU has a commit. */
gcmkONERROR(gckOS_Broadcast(Command->os,
hardware,
gcvBROADCAST_GPU_COMMIT));
/* Acquire the power management semaphore. */
gcmkONERROR(gckOS_AcquireSemaphore(Command->os,
Command->powerSemaphore));
semaAcquired = gcvTRUE;
}
/* Grab the conmmand queue mutex. */
gcmkONERROR(gckOS_AcquireMutex(Command->os,
Command->mutexQueue,
gcvINFINITE));
/* Success. */
gcmkFOOTER();
return gcvSTATUS_OK;
OnError:
if (semaAcquired)
{
/* Release the power management semaphore. */
gcmkVERIFY_OK(gckOS_ReleaseSemaphore(
Command->os, Command->powerSemaphore
));
}
if (atomIncremented)
{
/* Decrement the commit atom. */
gcmkVERIFY_OK(_IncrementCommitAtom(
Command, gcvFALSE
));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckCOMMAND_ExitCommit
**
** Release command queue synchronization objects.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to an gckCOMMAND object to destroy.
**
** gctBOOL FromPower
** Determines whether the call originates from inside the power
** management or not.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckCOMMAND_ExitCommit(
IN gckCOMMAND Command,
IN gctBOOL FromPower
)
{
gceSTATUS status;
gcmkHEADER_ARG("Command=0x%x", Command);
/* Release the power mutex. */
gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexQueue));
if (!FromPower)
{
/* Release the power management semaphore. */
gcmkONERROR(gckOS_ReleaseSemaphore(Command->os,
Command->powerSemaphore));
/* Decrement the commit atom. */
gcmkONERROR(_IncrementCommitAtom(Command, gcvFALSE));
}
/* Success. */
gcmkFOOTER();
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckCOMMAND_Start
**
** Start up the command queue.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to an gckCOMMAND object to start.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckCOMMAND_Start(
IN gckCOMMAND Command
)
{
gceSTATUS status;
gckHARDWARE hardware;
gctUINT32 waitOffset = 0;
gctUINT32 waitLinkBytes;
gctPOINTER logical;
gctUINT32 physical;
gctUINT32 address;
gcmkHEADER_ARG("Command=0x%x", Command);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
if (Command->running)
{
/* Command queue already running. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/* Extract the gckHARDWARE object. */
hardware = Command->kernel->hardware;
gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
/* Query the size of WAIT/LINK command sequence. */
gcmkONERROR(gckHARDWARE_WaitLink(
hardware,
gcvNULL,
~0U,
Command->offset,
&waitLinkBytes,
gcvNULL,
gcvNULL
));
if ((Command->pageSize - Command->offset < waitLinkBytes)
|| (Command->logical == gcvNULL)
)
{
/* Start at beginning of a new queue. */
gcmkONERROR(_NewQueue(Command, gcvTRUE));
}
logical = (gctUINT8_PTR) Command->logical + Command->offset;
physical = Command->physical + Command->offset;
address = Command->address + Command->offset;
/* Append WAIT/LINK. */
gcmkONERROR(gckHARDWARE_WaitLink(
hardware,
logical,
address,
0,
&waitLinkBytes,
&waitOffset,
&Command->waitSize
));
Command->waitLogical = (gctUINT8_PTR) logical + waitOffset;
Command->waitPhysical = physical + waitOffset;
Command->waitAddress = address + waitOffset;
#if gcdNONPAGED_MEMORY_CACHEABLE
/* Flush the cache for the wait/link. */
gcmkONERROR(gckOS_CacheClean(
Command->os,
Command->kernelProcessID,
gcvNULL,
physical,
logical,
waitLinkBytes
));
#endif
/* Adjust offset. */
Command->offset += waitLinkBytes;
Command->newQueue = gcvFALSE;
#if gcdSECURITY
/* Start FE by calling security service. */
gckKERNEL_SecurityStartCommand(
Command->kernel
);
#else
/* Enable command processor. */
gcmkONERROR(gckHARDWARE_Execute(
hardware,
address,
waitLinkBytes
));
#endif
/* Command queue is running. */
Command->running = gcvTRUE;
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckCOMMAND_Stop
**
** Stop the command queue.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to an gckCOMMAND object to stop.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckCOMMAND_Stop(
IN gckCOMMAND Command
)
{
gckHARDWARE hardware;
gceSTATUS status;
gctUINT32 idle;
gcmkHEADER_ARG("Command=0x%x", Command);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
if (!Command->running)
{
/* Command queue is not running. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/* Extract the gckHARDWARE object. */
hardware = Command->kernel->hardware;
gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
if (gckHARDWARE_IsFeatureAvailable(hardware,
gcvFEATURE_END_EVENT) == gcvSTATUS_TRUE)
{
/* Allocate the signal. */
if (Command->endEventSignal == gcvNULL)
{
gcmkONERROR(gckOS_CreateSignal(Command->os,
gcvTRUE,
&Command->endEventSignal));
}
/* Append the END EVENT command to trigger the signal. */
gcmkONERROR(gckEVENT_Stop(Command->kernel->eventObj,
Command->kernelProcessID,
Command->waitPhysical,
Command->waitLogical,
Command->waitAddress,
Command->endEventSignal,
&Command->waitSize));
}
else
{
/* Replace last WAIT with END. */
gcmkONERROR(gckHARDWARE_End(
hardware,
Command->waitLogical,
Command->waitAddress,
&Command->waitSize
));
#if USE_KERNEL_VIRTUAL_BUFFERS
if (hardware->kernel->virtualCommandBuffer)
{
gcmkONERROR(gckKERNEL_GetGPUAddress(
hardware->kernel,
Command->waitLogical,
gcvFALSE,
Command->virtualMemory,
&hardware->lastEnd
));
}
#endif
#if gcdSECURITY
gcmkONERROR(gckKERNEL_SecurityExecute(
Command->kernel, Command->waitLogical, 8
));
#endif
/* Update queue tail pointer. */
gcmkONERROR(gckHARDWARE_UpdateQueueTail(Command->kernel->hardware,
Command->logical,
Command->offset));
#if gcdNONPAGED_MEMORY_CACHEABLE
/* Flush the cache for the END. */
gcmkONERROR(gckOS_CacheClean(
Command->os,
Command->kernelProcessID,
gcvNULL,
(gctUINT32)Command->waitPhysical,
Command->waitLogical,
Command->waitSize
));
#endif
/* Wait for idle. */
gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle));
}
/* Command queue is no longer running. */
Command->running = gcvFALSE;
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckCOMMAND_Commit
**
** Commit a command buffer to the command queue.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to a gckCOMMAND object.
**
** gckCONTEXT Context
** Pointer to a gckCONTEXT object.
**
** gcoCMDBUF CommandBuffer
** Pointer to a gcoCMDBUF object.
**
** gcsSTATE_DELTA_PTR StateDelta
** Pointer to the state delta.
**
** gctUINT32 ProcessID
** Current process ID.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckCOMMAND_Commit(
IN gckCOMMAND Command,
IN gckCONTEXT Context,
IN gcoCMDBUF CommandBuffer,
IN gcsSTATE_DELTA_PTR StateDelta,
IN gctUINT32 ProcessID,
IN gctBOOL Shared,
IN gctUINT32 Index,
OUT gctUINT64_PTR CommitStamp,
OUT gctBOOL_PTR ContextSwitched
)
{
gceSTATUS status;
gctBOOL commitEntered = gcvFALSE;
gctBOOL contextAcquired = gcvFALSE;
gckHARDWARE hardware;
gctBOOL needCopy = gcvFALSE;
gctBOOL commandBufferMapped = gcvFALSE;
gcoCMDBUF commandBufferObject = gcvNULL;
gctBOOL stall = gcvFALSE;
gctBOOL contextSwitched = gcvFALSE;
#if !gcdNULL_DRIVER
gcsCONTEXT_PTR contextBuffer;
struct _gcoCMDBUF _commandBufferObject;
gctPHYS_ADDR_T commandBufferPhysical;
gctUINT8_PTR commandBufferLogical = gcvNULL;
gctUINT32 commandBufferAddress = 0;
gctUINT8_PTR commandBufferTail = gcvNULL;
gctUINT commandBufferSize;
gctSIZE_T nopBytes;
gctUINT32 pipeBytes;
gctUINT32 linkBytes;
gctSIZE_T bytes;
gctUINT32 offset;
#if gcdNONPAGED_MEMORY_CACHEABLE
gctPHYS_ADDR entryPhysical;
#endif
gctPOINTER entryLogical;
gctUINT32 entryAddress;
gctUINT32 entryBytes;
#if gcdNONPAGED_MEMORY_CACHEABLE
gctPHYS_ADDR exitPhysical;
gctPOINTER exitLogical;
#endif
gctUINT32 exitAddress;
gctUINT32 exitBytes;
gctUINT32 waitLinkPhysical;
gctPOINTER waitLinkLogical;
gctUINT32 waitLinkAddress;
gctUINT32 waitLinkBytes;
gctUINT32 waitOffset;
gctUINT32 waitSize;
#ifdef __QNXNTO__
gctPOINTER userCommandBufferLogical = gcvNULL;
gctBOOL userCommandBufferLogicalMapped = gcvFALSE;
#endif
#if gcdPROCESS_ADDRESS_SPACE
gckMMU mmu;
gctUINT32 oldValue;
#endif
#if gcdDUMP_COMMAND
gctPOINTER contextDumpLogical = gcvNULL;
gctSIZE_T contextDumpBytes = 0;
gctPOINTER bufferDumpLogical = gcvNULL;
gctSIZE_T bufferDumpBytes = 0;
# endif
gctUINT32 exitLinkLow = 0, exitLinkHigh = 0;
gctUINT32 entryLinkLow = 0, entryLinkHigh = 0;
gctUINT32 commandLinkLow = 0, commandLinkHigh = 0;
gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer = gcvNULL;
gctUINT64 asyncCommandStamp = 0;
gcoCMDBUF lastCommandBuffer = gcvNULL;
gctPOINTER pointer = gcvNULL;
#endif
gcmkHEADER_ARG(
"Command=0x%x CommandBuffer=0x%x ProcessID=%d",
Command, CommandBuffer, ProcessID
);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
#if !gcdNULL_DRIVER
gcmkONERROR(_ProcessUserCommandBufferList(
Command,
CommandBuffer,
&lastCommandBuffer
));
#endif
#if gcdPROCESS_ADDRESS_SPACE
gcmkONERROR(gckKERNEL_GetProcessMMU(Command->kernel, &mmu));
gcmkONERROR(gckOS_AtomicExchange(Command->os,
mmu->pageTableDirty[Command->kernel->core],
0,
&oldValue));
#else
#endif
/* Acquire the command queue. */
gcmkONERROR(gckCOMMAND_EnterCommit(Command, gcvFALSE));
commitEntered = gcvTRUE;
/* Acquire the context switching mutex. */
gcmkONERROR(gckOS_AcquireMutex(
Command->os, Command->mutexContext, gcvINFINITE
));
contextAcquired = gcvTRUE;
/* Extract the gckHARDWARE and gckEVENT objects. */
hardware = Command->kernel->hardware;
/* Check wehther we need to copy the structures or not. */
gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy));
#if gcdNULL_DRIVER
/* Context switch required? */
if ((Context != gcvNULL) && (Command->currContext != Context))
{
/* Yes, merge in the deltas. */
gckCONTEXT_Update(Context, ProcessID, StateDelta);
/* Update the current context. */
Command->currContext = Context;
contextSwitched = gcvTRUE;
}
#else
if (needCopy)
{
commandBufferObject = &_commandBufferObject;
gcmkONERROR(gckOS_CopyFromUserData(
Command->os,
commandBufferObject,
CommandBuffer,
gcmSIZEOF(struct _gcoCMDBUF)
));
gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER);
}
else
{
gcmkONERROR(gckOS_MapUserPointer(
Command->os,
CommandBuffer,
gcmSIZEOF(struct _gcoCMDBUF),
&pointer
));
commandBufferObject = pointer;
gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER);
commandBufferMapped = gcvTRUE;
}
gcmkONERROR(_HandlePatchList(Command, commandBufferObject, needCopy, &asyncCommandStamp));
/* Query the size of NOP command. */
gcmkONERROR(gckHARDWARE_Nop(
hardware, gcvNULL, &nopBytes
));
/* Query the size of pipe select command sequence. */
gcmkONERROR(gckHARDWARE_PipeSelect(
hardware, gcvNULL, gcvPIPE_3D, &pipeBytes
));
/* Query the size of LINK command. */
gcmkONERROR(gckHARDWARE_Link(
hardware, gcvNULL, 0, 0, &linkBytes, gcvNULL, gcvNULL
));
/* Compute the command buffer entry and the size. */
commandBufferLogical
= (gctUINT8_PTR) gcmUINT64_TO_PTR(commandBufferObject->logical)
+ commandBufferObject->startOffset;
/* Get the hardware address. */
if (Command->kernel->virtualCommandBuffer)
{
gckKERNEL kernel = Command->kernel;
virtualCommandBuffer = gcmNAME_TO_PTR(commandBufferObject->physical);
if (virtualCommandBuffer == gcvNULL)
{
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
gcmkONERROR(gckKERNEL_GetGPUAddress(
Command->kernel,
commandBufferLogical,
gcvTRUE,
virtualCommandBuffer,
&commandBufferAddress
));
}
else
{
gcmkONERROR(gckHARDWARE_ConvertLogical(
hardware,
commandBufferLogical,
gcvTRUE,
&commandBufferAddress
));
}
#ifdef __QNXNTO__
userCommandBufferLogical = (gctPOINTER) commandBufferLogical;
gcmkONERROR(gckOS_MapUserPointer(
Command->os,
userCommandBufferLogical,
0,
&pointer));
commandBufferLogical = pointer;
userCommandBufferLogicalMapped = gcvTRUE;
gcmkONERROR(gckOS_GetPhysicalAddress(
Command->os,
commandBufferLogical,
&commandBufferPhysical
));
#else
/* Get the physical address. */
gcmkONERROR(gckOS_UserLogicalToPhysical(
Command->os,
commandBufferLogical,
&commandBufferPhysical
));
#endif
commandBufferSize
= commandBufferObject->offset
+ commandBufferObject->reservedTail
- commandBufferObject->startOffset;
gcmkONERROR(_FlushMMU(Command));
if (Command->dummyDraw == gcvTRUE &&
Context != gcvNULL)
{
Command->dummyDraw = gcvFALSE;
gcmkONERROR(_DummyDraw(Command));
}
if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT) && asyncCommandStamp != 0)
{
gcmkONERROR(_WaitForAsyncCommandStamp(Command, asyncCommandStamp));
}
/* Get the current offset. */
offset = Command->offset;
/* Compute number of bytes left in current kernel command queue. */
bytes = Command->pageSize - offset;
/* Query the size of WAIT/LINK command sequence. */
gcmkONERROR(gckHARDWARE_WaitLink(
hardware,
gcvNULL,
~0U,
offset,
&waitLinkBytes,
gcvNULL,
gcvNULL
));
/* Is there enough space in the current command queue? */
if (bytes < waitLinkBytes)
{
/* No, create a new one. */
gcmkONERROR(_NewQueue(Command, gcvFALSE));
/* Get the new current offset. */
offset = Command->offset;
/* Recompute the number of bytes in the new kernel command queue. */
bytes = Command->pageSize - offset;
gcmkASSERT(bytes >= waitLinkBytes);
}
/* Compute the location if WAIT/LINK command sequence. */
waitLinkPhysical = Command->physical + offset;
waitLinkLogical = (gctUINT8_PTR) Command->logical + offset;
waitLinkAddress = Command->address + offset;
/* Context switch required? */
if (Context == gcvNULL)
{
/* See if we have to switch pipes for the command buffer. */
if (commandBufferObject->entryPipe == Command->pipeSelect)
{
/* Skip pipe switching sequence. */
offset = pipeBytes;
}
else
{
/* The current hardware and the entry command buffer pipes
** are different, switch to the correct pipe. */
gcmkONERROR(gckHARDWARE_PipeSelect(
Command->kernel->hardware,
commandBufferLogical,
commandBufferObject->entryPipe,
&pipeBytes
));
/* Do not skip pipe switching sequence. */
offset = 0;
}
/* Compute the entry. */
#if gcdNONPAGED_MEMORY_CACHEABLE
entryPhysical = (gctUINT8_PTR) commandBufferPhysical + offset;
#endif
entryLogical = commandBufferLogical + offset;
entryAddress = commandBufferAddress + offset;
entryBytes = commandBufferSize - offset;
Command->currContext = gcvNULL;
}
#if gcdDEBUG_OPTION && gcdDEBUG_FORCE_CONTEXT_UPDATE
else if (1)
#else
else if (Command->currContext != Context)
#endif
{
/* Get the current context buffer. */
contextBuffer = Context->buffer;
/* Yes, merge in the deltas. */
gcmkONERROR(gckCONTEXT_Update(Context, ProcessID, StateDelta));
contextSwitched = gcvTRUE;
/***************************************************************
** SWITCHING CONTEXT.
*/
/* Determine context buffer entry offset. */
offset = (Command->pipeSelect == gcvPIPE_3D)
/* Skip pipe switching sequence. */
? Context->entryOffset3D + Context->pipeSelectBytes
/* Do not skip pipe switching sequence. */
: Context->entryOffset3D;
/* Compute the entry. */
#if gcdNONPAGED_MEMORY_CACHEABLE
entryPhysical = (gctUINT8_PTR) contextBuffer->physical + offset;
#endif
entryLogical = (gctUINT8_PTR) contextBuffer->logical + offset;
entryAddress = contextBuffer->address + offset;
entryBytes = Context->bufferSize - offset;
/* See if we have to switch pipes between the context
and command buffers. */
if (commandBufferObject->entryPipe == gcvPIPE_3D)
{
/* Skip pipe switching sequence. */
offset = pipeBytes;
}
else
{
/* The current hardware and the initial context pipes are
different, switch to the correct pipe. */
gcmkONERROR(gckHARDWARE_PipeSelect(
Command->kernel->hardware,
commandBufferLogical,
commandBufferObject->entryPipe,
&pipeBytes
));
/* Do not skip pipe switching sequence. */
offset = 0;
}
/* Generate a LINK from the context buffer to
the command buffer. */
gcmkONERROR(gckHARDWARE_Link(
hardware,
contextBuffer->link3D,
commandBufferAddress + offset,
commandBufferSize - offset,
&linkBytes,
&commandLinkLow,
&commandLinkHigh
));
#if gcdNONPAGED_MEMORY_CACHEABLE
/* Flush the context buffer cache. */
gcmkONERROR(gckOS_CacheClean(
Command->os,
Command->kernelProcessID,
gcvNULL,
(gctUINT32)entryPhysical,
entryLogical,
entryBytes
));
#endif
/* Update the current context. */
Command->currContext = Context;
#if gcdDUMP_COMMAND
contextDumpLogical = entryLogical;
contextDumpBytes = entryBytes;
#endif
#if gcdSECURITY
/* Commit context buffer to trust zone. */
gckKERNEL_SecurityExecute(
Command->kernel,
entryLogical,
entryBytes - 8
);
#endif
#if gcdRECORD_COMMAND
gckRECORDER_Record(
Command->recorder,
gcvNULL,
0xFFFFFFFF,
entryLogical,
entryBytes
);
#endif
}
/* Same context. */
else
{
/* See if we have to switch pipes for the command buffer. */
if (commandBufferObject->entryPipe == Command->pipeSelect)
{
/* Skip pipe switching sequence. */
offset = pipeBytes;
}
else
{
/* The current hardware and the entry command buffer pipes
** are different, switch to the correct pipe. */
gcmkONERROR(gckHARDWARE_PipeSelect(
Command->kernel->hardware,
commandBufferLogical,
commandBufferObject->entryPipe,
&pipeBytes
));
/* Do not skip pipe switching sequence. */
offset = 0;
}
/* Compute the entry. */
#if gcdNONPAGED_MEMORY_CACHEABLE
entryPhysical = (gctUINT8_PTR) commandBufferPhysical + offset;
#endif
entryLogical = commandBufferLogical + offset;
entryAddress = commandBufferAddress + offset;
entryBytes = commandBufferSize - offset;
}
#if gcdDUMP_COMMAND
bufferDumpLogical = commandBufferLogical + offset;
bufferDumpBytes = commandBufferSize - offset;
#endif
#if gcdSECURE_USER
/* Process user hints. */
gcmkONERROR(_ProcessHints(Command, ProcessID, commandBufferObject));
#endif
/* Determine the location to jump to for the command buffer being
** scheduled. */
if (Command->newQueue)
{
/* New command queue, jump to the beginning of it. */
#if gcdNONPAGED_MEMORY_CACHEABLE
exitPhysical = Command->physical;
exitLogical = Command->logical;
#endif
exitAddress = Command->address;
exitBytes = Command->offset + waitLinkBytes;
}
else
{
/* Still within the preexisting command queue, jump to the new
WAIT/LINK command sequence. */
#if gcdNONPAGED_MEMORY_CACHEABLE
exitPhysical = waitLinkPhysical;
exitLogical = waitLinkLogical;
#endif
exitAddress = waitLinkAddress;
exitBytes = waitLinkBytes;
}
/* Add a new WAIT/LINK command sequence. When the command buffer which is
currently being scheduled is fully executed by the GPU, the FE will
jump to this WAIT/LINK sequence. */
gcmkONERROR(gckHARDWARE_WaitLink(
hardware,
waitLinkLogical,
waitLinkAddress,
offset,
&waitLinkBytes,
&waitOffset,
&waitSize
));
#if gcdNONPAGED_MEMORY_CACHEABLE
/* Flush the command queue cache. */
gcmkONERROR(gckOS_CacheClean(
Command->os,
Command->kernelProcessID,
gcvNULL,
(gctUINT32)exitPhysical,
exitLogical,
exitBytes
));
#endif
/* Determine the location of the TAIL in the command buffer. */
commandBufferTail
= commandBufferLogical
+ commandBufferSize
- commandBufferObject->reservedTail;
/* Generate command which writes out commit stamp. */
if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT))
{
gctUINT32 bytes;
gcmkONERROR(gckHARDWARE_Fence(
hardware,
gcvENGINE_RENDER,
commandBufferTail,
Command->fence->address,
Command->commitStamp,
&bytes
));
commandBufferTail += gcdRENDER_FENCE_LENGTH;
}
/* Generate a LINK from the end of the command buffer being scheduled
back to the kernel command queue. */
#if !gcdSECURITY
if (Shared == gcvFALSE)
{
gcmkONERROR(gckHARDWARE_Link(
hardware,
commandBufferTail,
exitAddress,
exitBytes,
&linkBytes,
&exitLinkLow,
&exitLinkHigh
));
}
else
{
gctUINT8_PTR link = commandBufferTail + Index * 16;
gctSIZE_T bytes = 8;
gcmkONERROR(gckHARDWARE_ChipEnable(
hardware,
link,
(gceCORE_3D_MASK)(1 << hardware->kernel->chipID),
&bytes
));
link += bytes;
gcmkONERROR(gckHARDWARE_Link(
hardware,
link,
exitAddress,
exitBytes,
&linkBytes,
&exitLinkLow,
&exitLinkHigh
));
link += linkBytes;
}
#endif
#if gcdNONPAGED_MEMORY_CACHEABLE
/* Flush the command buffer cache. */
gcmkONERROR(gckOS_CacheClean(
Command->os,
ProcessID,
gcvNULL,
(gctUINT32)commandBufferPhysical,
commandBufferLogical,
commandBufferSize
));
#endif
#if gcdRECORD_COMMAND
gckRECORDER_Record(
Command->recorder,
commandBufferLogical + offset,
commandBufferSize - offset,
gcvNULL,
0xFFFFFFFF
);
gckRECORDER_AdvanceIndex(Command->recorder, Command->commitStamp);
#endif
#if gcdSECURITY
/* Submit command buffer to trust zone. */
gckKERNEL_SecurityExecute(
Command->kernel,
commandBufferLogical + offset,
commandBufferSize - offset - 8
);
#else
/* Generate a LINK from the previous WAIT/LINK command sequence to the
entry determined above (either the context or the command buffer).
This LINK replaces the WAIT instruction from the previous WAIT/LINK
pair, therefore we use WAIT metrics for generation of this LINK.
This action will execute the entire sequence. */
gcmkONERROR(gckHARDWARE_Link(
hardware,
Command->waitLogical,
entryAddress,
entryBytes,
&Command->waitSize,
&entryLinkLow,
&entryLinkHigh
));
#endif
#if gcdLINK_QUEUE_SIZE
if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_USER_COMMAND)
{
gcuQUEUEDATA data;
gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid));
data.linkData.start = entryAddress;
data.linkData.end = entryAddress + entryBytes;
data.linkData.linkLow = entryLinkLow;
data.linkData.linkHigh = entryLinkHigh;
gckQUEUE_Enqueue(&hardware->linkQueue, &data);
if (commandBufferAddress + offset != entryAddress)
{
data.linkData.start = commandBufferAddress + offset;
data.linkData.end = commandBufferAddress + commandBufferSize;
data.linkData.linkLow = commandLinkLow;
data.linkData.linkHigh = commandLinkHigh;
gckQUEUE_Enqueue(&hardware->linkQueue, &data);
}
if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND)
{
data.linkData.start = exitAddress;
data.linkData.end = exitAddress + exitBytes;
data.linkData.linkLow = exitLinkLow;
data.linkData.linkHigh = exitLinkHigh;
/* Dump kernel command.*/
gckQUEUE_Enqueue(&hardware->linkQueue, &data);
}
}
#endif
#if gcdNONPAGED_MEMORY_CACHEABLE
/* Flush the cache for the link. */
gcmkONERROR(gckOS_CacheClean(
Command->os,
Command->kernelProcessID,
gcvNULL,
(gctUINT32)Command->waitPhysical,
Command->waitLogical,
Command->waitSize
));
#endif
gcmkDUMPCOMMAND(
Command->os,
Command->waitLogical,
Command->waitSize,
gcvDUMP_BUFFER_LINK,
gcvFALSE
);
gcmkDUMPCOMMAND(
Command->os,
contextDumpLogical,
contextDumpBytes,
gcvDUMP_BUFFER_CONTEXT,
gcvFALSE
);
gcmkDUMPCOMMAND(
Command->os,
bufferDumpLogical,
bufferDumpBytes,
gcvDUMP_BUFFER_USER,
gcvFALSE
);
gcmkDUMPCOMMAND(
Command->os,
waitLinkLogical,
waitLinkBytes,
gcvDUMP_BUFFER_WAITLINK,
gcvFALSE
);
/* Update the current pipe. */
Command->pipeSelect = commandBufferObject->exitPipe;
/* Update command queue offset. */
Command->offset += waitLinkBytes;
Command->newQueue = gcvFALSE;
/* Update address of last WAIT. */
Command->waitPhysical = waitLinkPhysical + waitOffset;
Command->waitLogical = (gctUINT8_PTR)waitLinkLogical + waitOffset;
Command->waitAddress = waitLinkAddress + waitOffset;
Command->waitSize = waitSize;
/* Update queue tail pointer. */
gcmkONERROR(gckHARDWARE_UpdateQueueTail(
hardware, Command->logical, Command->offset
));
#if gcdDUMP_COMMAND
gcmkPRINT("@[kernel.commit]");
#endif
#endif /* gcdNULL_DRIVER */
/* Release the context switching mutex. */
gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
contextAcquired = gcvFALSE;
*CommitStamp = Command->commitStamp;
*ContextSwitched = contextSwitched;
Command->commitStamp++;
stall = gcvFALSE;
#if gcdLINK_QUEUE_SIZE
if (Command->kernel->stuckDump == gcvSTUCK_DUMP_STALL_COMMAND)
{
if ((Command->commitStamp % (gcdLINK_QUEUE_SIZE/2)) == 0)
{
/* If only context buffer and command buffer is recorded,
** each commit costs 2 slot in queue, to make sure command
** causing stuck is recorded, number of pending command buffer
** is limited to (gckLINK_QUEUE_SIZE/2)
*/
stall = gcvTRUE;
}
}
#endif
/* Release the command queue. */
gcmkONERROR(gckCOMMAND_ExitCommit(Command, gcvFALSE));
commitEntered = gcvFALSE;
if ((Command->kernel->hardware->options.gpuProfiler == gcvTRUE) &&
(Command->kernel->profileEnable == gcvTRUE))
{
gcmkONERROR(gckCOMMAND_Stall(Command, gcvTRUE));
if (Command->currContext)
{
gcmkONERROR(gckHARDWARE_UpdateContextProfile(
hardware,
Command->currContext));
}
}
if (status == gcvSTATUS_INTERRUPTED)
{
gcmkTRACE(
gcvLEVEL_INFO,
"%s(%d): Intterupted in gckEVENT_Submit",
__FUNCTION__, __LINE__
);
status = gcvSTATUS_OK;
}
else
{
gcmkONERROR(status);
}
#ifdef __QNXNTO__
if (userCommandBufferLogicalMapped)
{
gcmkONERROR(gckOS_UnmapUserPointer(
Command->os,
userCommandBufferLogical,
0,
commandBufferLogical));
userCommandBufferLogicalMapped = gcvFALSE;
}
#endif
/* Unmap the command buffer pointer. */
if (commandBufferMapped)
{
gcmkONERROR(gckOS_UnmapUserPointer(
Command->os,
CommandBuffer,
gcmSIZEOF(struct _gcoCMDBUF),
commandBufferObject
));
commandBufferMapped = gcvFALSE;
}
/* Return status. */
gcmkFOOTER();
return gcvSTATUS_OK;
OnError:
if (contextAcquired)
{
/* Release the context switching mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
}
if (commitEntered)
{
/* Release the command queue mutex. */
gcmkVERIFY_OK(gckCOMMAND_ExitCommit(Command, gcvFALSE));
}
#ifdef __QNXNTO__
if (userCommandBufferLogicalMapped)
{
gcmkVERIFY_OK(gckOS_UnmapUserPointer(
Command->os,
userCommandBufferLogical,
0,
commandBufferLogical));
}
#endif
/* Unmap the command buffer pointer. */
if (commandBufferMapped)
{
gcmkVERIFY_OK(gckOS_UnmapUserPointer(
Command->os,
CommandBuffer,
gcmSIZEOF(struct _gcoCMDBUF),
commandBufferObject
));
}
/* Return status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckCOMMAND_Reserve
**
** Reserve space in the command queue. Also acquire the command queue mutex.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to an gckCOMMAND object.
**
** gctSIZE_T RequestedBytes
** Number of bytes previously reserved.
**
** OUTPUT:
**
** gctPOINTER * Buffer
** Pointer to a variable that will receive the address of the reserved
** space.
**
** gctSIZE_T * BufferSize
** Pointer to a variable that will receive the number of bytes
** available in the command queue.
*/
gceSTATUS
gckCOMMAND_Reserve(
IN gckCOMMAND Command,
IN gctUINT32 RequestedBytes,
OUT gctPOINTER * Buffer,
OUT gctUINT32 * BufferSize
)
{
gceSTATUS status;
gctUINT32 bytes;
gctUINT32 requiredBytes;
gctUINT32 requestedAligned;
gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
/* Compute aligned number of reuested bytes. */
requestedAligned = gcmALIGN(RequestedBytes, Command->alignment);
/* Another WAIT/LINK command sequence will have to be appended after
the requested area being reserved. Compute the number of bytes
required for WAIT/LINK at the location after the reserved area. */
gcmkONERROR(gckHARDWARE_WaitLink(
Command->kernel->hardware,
gcvNULL,
~0U,
Command->offset + requestedAligned,
&requiredBytes,
gcvNULL,
gcvNULL
));
/* Compute total number of bytes required. */
requiredBytes += requestedAligned;
/* Compute number of bytes available in command queue. */
bytes = Command->pageSize - Command->offset;
/* Is there enough space in the current command queue? */
if (bytes < requiredBytes)
{
/* Create a new command queue. */
gcmkONERROR(_NewQueue(Command, gcvFALSE));
/* Recompute the number of bytes in the new kernel command queue. */
bytes = Command->pageSize - Command->offset;
/* Still not enough space? */
if (bytes < requiredBytes)
{
/* Rare case, not enough room in command queue. */
gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL);
}
}
/* Return pointer to empty slot command queue. */
*Buffer = (gctUINT8 *) Command->logical + Command->offset;
/* Return number of bytes left in command queue. */
*BufferSize = bytes;
/* Success. */
gcmkFOOTER_ARG("*Buffer=0x%x *BufferSize=%lu", *Buffer, *BufferSize);
return gcvSTATUS_OK;
OnError:
/* Return status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckCOMMAND_Execute
**
** Execute a previously reserved command queue by appending a WAIT/LINK command
** sequence after it and modifying the last WAIT into a LINK command. The
** command FIFO mutex will be released whether this function succeeds or not.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to an gckCOMMAND object.
**
** gctSIZE_T RequestedBytes
** Number of bytes previously reserved.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckCOMMAND_Execute(
IN gckCOMMAND Command,
IN gctUINT32 RequestedBytes
)
{
gceSTATUS status;
gctUINT32 waitLinkPhysical;
gctUINT8_PTR waitLinkLogical;
gctUINT32 waitLinkAddress;
gctUINT32 waitLinkOffset;
gctUINT32 waitLinkBytes;
gctUINT32 waitPhysical;
gctPOINTER waitLogical;
gctUINT32 waitAddress;
gctUINT32 waitOffset;
gctUINT32 waitBytes;
gctUINT32 linkLow, linkHigh;
#if gcdNONPAGED_MEMORY_CACHEABLE
gctPHYS_ADDR execPhysical;
#endif
gctPOINTER execLogical;
gctUINT32 execAddress;
gctUINT32 execBytes;
gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
/* Compute offset for WAIT/LINK. */
waitLinkOffset = Command->offset + RequestedBytes;
/* Compute number of bytes left in command queue. */
waitLinkBytes = Command->pageSize - waitLinkOffset;
/* Compute the location if WAIT/LINK command sequence. */
waitLinkPhysical = Command->physical + waitLinkOffset;
waitLinkLogical = (gctUINT8_PTR) Command->logical + waitLinkOffset;
waitLinkAddress = Command->address + waitLinkOffset;
/* Append WAIT/LINK in command queue. */
gcmkONERROR(gckHARDWARE_WaitLink(
Command->kernel->hardware,
waitLinkLogical,
waitLinkAddress,
waitLinkOffset,
&waitLinkBytes,
&waitOffset,
&waitBytes
));
/* Compute the location if WAIT command. */
waitPhysical = waitLinkPhysical + waitOffset;
waitLogical = waitLinkLogical + waitOffset;
waitAddress = waitLinkAddress + waitOffset;
/* Determine the location to jump to for the command buffer being
** scheduled. */
if (Command->newQueue)
{
/* New command queue, jump to the beginning of it. */
#if gcdNONPAGED_MEMORY_CACHEABLE
execPhysical = Command->physical;
#endif
execLogical = Command->logical;
execAddress = Command->address;
execBytes = waitLinkOffset + waitLinkBytes;
}
else
{
/* Still within the preexisting command queue, jump directly to the
reserved area. */
#if gcdNONPAGED_MEMORY_CACHEABLE
execPhysical = (gctUINT8 *) Command->physical + Command->offset;
#endif
execLogical = (gctUINT8 *) Command->logical + Command->offset;
execAddress = Command->address + Command->offset;
execBytes = RequestedBytes + waitLinkBytes;
}
#if gcdNONPAGED_MEMORY_CACHEABLE
/* Flush the cache. */
gcmkONERROR(gckOS_CacheClean(
Command->os,
Command->kernelProcessID,
gcvNULL,
(gctUINT32)execPhysical,
execLogical,
execBytes
));
#endif
/* Convert the last WAIT into a LINK. */
gcmkONERROR(gckHARDWARE_Link(
Command->kernel->hardware,
Command->waitLogical,
execAddress,
execBytes,
&Command->waitSize,
&linkLow,
&linkHigh
));
#if gcdNONPAGED_MEMORY_CACHEABLE
/* Flush the cache. */
gcmkONERROR(gckOS_CacheClean(
Command->os,
Command->kernelProcessID,
gcvNULL,
(gctUINT32)Command->waitPhysical,
Command->waitLogical,
Command->waitSize
));
#endif
#if gcdLINK_QUEUE_SIZE
if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND)
{
gcuQUEUEDATA data;
gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid));
data.linkData.start = execAddress;
data.linkData.end = execAddress + execBytes;
data.linkData.linkLow = linkLow;
data.linkData.linkHigh = linkHigh;
gckQUEUE_Enqueue(&Command->kernel->hardware->linkQueue, &data);
}
#endif
gcmkDUMPCOMMAND(
Command->os,
Command->waitLogical,
Command->waitSize,
gcvDUMP_BUFFER_LINK,
gcvFALSE
);
gcmkDUMPCOMMAND(
Command->os,
execLogical,
execBytes,
gcvDUMP_BUFFER_KERNEL,
gcvFALSE
);
/* Update the pointer to the last WAIT. */
Command->waitPhysical = waitPhysical;
Command->waitLogical = waitLogical;
Command->waitAddress = waitAddress;
Command->waitSize = waitBytes;
/* Update the command queue. */
Command->offset += RequestedBytes + waitLinkBytes;
Command->newQueue = gcvFALSE;
/* Update queue tail pointer. */
gcmkONERROR(gckHARDWARE_UpdateQueueTail(
Command->kernel->hardware, Command->logical, Command->offset
));
#if gcdDUMP_COMMAND
gcmkPRINT("@[kernel.execute]");
#endif
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckCOMMAND_Stall
**
** The calling thread will be suspended until the command queue has been
** completed.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to an gckCOMMAND object.
**
** gctBOOL FromPower
** Determines whether the call originates from inside the power
** management or not.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckCOMMAND_Stall(
IN gckCOMMAND Command,
IN gctBOOL FromPower
)
{
#if gcdNULL_DRIVER
/* Do nothing with infinite hardware. */
return gcvSTATUS_OK;
#else
gckOS os;
gckHARDWARE hardware;
gckEVENT eventObject;
gceSTATUS status;
gctSIGNAL signal = gcvNULL;
gctUINT timer = 0;
gcmkHEADER_ARG("Command=0x%x", Command);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
/* Extract the gckOS object pointer. */
os = Command->os;
gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
/* Extract the gckHARDWARE object pointer. */
hardware = Command->kernel->hardware;
gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
/* Extract the gckEVENT object pointer. */
eventObject = Command->kernel->eventObj;
gcmkVERIFY_OBJECT(eventObject, gcvOBJ_EVENT);
/* Allocate the signal. */
gcmkONERROR(gckOS_CreateSignal(os, gcvTRUE, &signal));
/* Append the EVENT command to trigger the signal. */
gcmkONERROR(gckEVENT_Signal(eventObject, signal, gcvKERNEL_PIXEL));
/* Submit the event queue. */
gcmkONERROR(gckEVENT_Submit(eventObject, gcvTRUE, FromPower));
#if gcdDUMP_COMMAND
gcmkPRINT("@[kernel.stall]");
#endif
if (status == gcvSTATUS_CHIP_NOT_READY)
{
/* Error. */
goto OnError;
}
do
{
/* Wait for the signal. */
status = gckOS_WaitSignal(os, signal, gcvTRUE, gcdGPU_ADVANCETIMER);
if (status == gcvSTATUS_TIMEOUT)
{
#if gcmIS_DEBUG(gcdDEBUG_CODE)
gctUINT32 idle;
/* Read idle register. */
gcmkVERIFY_OK(gckHARDWARE_GetIdle(
hardware, gcvFALSE, &idle
));
gcmkTRACE(
gcvLEVEL_ERROR,
"%s(%d): idle=%08x",
__FUNCTION__, __LINE__, idle
);
gcmkVERIFY_OK(gckOS_MemoryBarrier(os, gcvNULL));
#endif
/* Advance timer. */
timer += gcdGPU_ADVANCETIMER;
}
else if (status == gcvSTATUS_INTERRUPTED)
{
gcmkONERROR(gcvSTATUS_INTERRUPTED);
}
}
while (gcmIS_ERROR(status));
/* Bail out on timeout. */
if (gcmIS_ERROR(status))
{
/* Broadcast the stuck GPU. */
gcmkONERROR(gckOS_Broadcast(
os, hardware, gcvBROADCAST_GPU_STUCK
));
}
/* Delete the signal. */
gcmkVERIFY_OK(gckOS_DestroySignal(os, signal));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (signal != gcvNULL)
{
/* Free the signal. */
gcmkVERIFY_OK(gckOS_DestroySignal(os, signal));
}
/* Return the status. */
gcmkFOOTER();
return status;
#endif
}
/*******************************************************************************
**
** gckCOMMAND_Attach
**
** Attach user process.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to a gckCOMMAND object.
**
** gctUINT32 ProcessID
** Current process ID.
**
** OUTPUT:
**
** gckCONTEXT * Context
** Pointer to a variable that will receive a pointer to a new
** gckCONTEXT object.
**
** gctSIZE_T * StateCount
** Pointer to a variable that will receive the number of states
** in the context buffer.
*/
#if (gcdENABLE_3D || gcdENABLE_2D)
gceSTATUS
gckCOMMAND_Attach(
IN gckCOMMAND Command,
OUT gckCONTEXT * Context,
OUT gctSIZE_T * MaxState,
OUT gctUINT32 * NumStates,
IN gctUINT32 ProcessID
)
{
gceSTATUS status;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Command=0x%x", Command);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
/* Acquire the context switching mutex. */
gcmkONERROR(gckOS_AcquireMutex(
Command->os, Command->mutexContext, gcvINFINITE
));
acquired = gcvTRUE;
/* Construct a gckCONTEXT object. */
gcmkONERROR(gckCONTEXT_Construct(
Command->os,
Command->kernel->hardware,
ProcessID,
Context
));
/* Return the number of states in the context. */
* MaxState = (* Context)->maxState;
* NumStates = (* Context)->numStates;
/* Release the context switching mutex. */
gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
acquired = gcvFALSE;
/* Success. */
gcmkFOOTER_ARG("*Context=0x%x", *Context);
return gcvSTATUS_OK;
OnError:
/* Release mutex. */
if (acquired)
{
/* Release the context switching mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
acquired = gcvFALSE;
}
/* Return the status. */
gcmkFOOTER();
return status;
}
#endif
/*******************************************************************************
**
** gckCOMMAND_Detach
**
** Detach user process.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to a gckCOMMAND object.
**
** gckCONTEXT Context
** Pointer to a gckCONTEXT object to be destroyed.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckCOMMAND_Detach(
IN gckCOMMAND Command,
IN gckCONTEXT Context
)
{
gceSTATUS status;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Command=0x%x Context=0x%x", Command, Context);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
/* Acquire the context switching mutex. */
gcmkONERROR(gckOS_AcquireMutex(
Command->os, Command->mutexContext, gcvINFINITE
));
acquired = gcvTRUE;
/* Construct a gckCONTEXT object. */
gcmkONERROR(gckCONTEXT_Destroy(Context));
if (Command->currContext == Context)
{
/* Detach from gckCOMMAND object if the destoryed context is current context. */
Command->currContext = gcvNULL;
}
/* Release the context switching mutex. */
gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
acquired = gcvFALSE;
/* Return the status. */
gcmkFOOTER();
return gcvSTATUS_OK;
OnError:
/* Release mutex. */
if (acquired)
{
/* Release the context switching mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
acquired = gcvFALSE;
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckCOMMAND_DumpExecutingBuffer
**
** Dump the command buffer which GPU is executing.
**
** INPUT:
**
** gckCOMMAND Command
** Pointer to a gckCOMMAND object.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckCOMMAND_DumpExecutingBuffer(
IN gckCOMMAND Command
)
{
gceSTATUS status;
gckVIRTUAL_COMMAND_BUFFER_PTR buffer = gcvNULL;
gctUINT32 gpuAddress;
gctSIZE_T pageCount;
gctPOINTER entry = gcvNULL;
gckOS os = Command->os;
gckKERNEL kernel = Command->kernel;
gctUINT32 i;
gctUINT32 dumpRear;
gckQUEUE queue = &kernel->hardware->linkQueue;
gctSIZE_T bytes;
gckLINKDATA linkData;
gcuQUEUEDATA * queueData;
gctUINT32 offset;
gctPOINTER entryDump;
gctUINT32 pid;
gctUINT8 processName[24] = {0};
gctPHYS_ADDR_T cpuPhysical;
gcmkPRINT("**************************\n");
gcmkPRINT("**** COMMAND BUF DUMP ****\n");
gcmkPRINT("**************************\n");
gcmkPRINT(" Submitted commit stamp = %lld", Command->commitStamp - 1);
gcmkPRINT(" Executed commit stamp = %lld", *(gctUINT64_PTR)Command->fence->logical);
gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress));
gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress));
gcmkPRINT("DMA Address 0x%08X, memory around:", gpuAddress);
/* Search and dump memory around DMA address. */
if (kernel->virtualCommandBuffer)
{
status = gckDEVICE_QueryGPUAddress(kernel->device, kernel, gpuAddress, &buffer);
}
else
{
status = gcvSTATUS_OK;
}
if (gcmIS_SUCCESS(status))
{
if (kernel->virtualCommandBuffer)
{
gcmkVERIFY_OK(gckOS_CreateKernelVirtualMapping(
os, buffer->virtualBuffer.physical, buffer->virtualBuffer.bytes, &entry, &pageCount));
offset = gpuAddress - buffer->virtualBuffer.gpuAddress;
entryDump = entry;
/* Dump one pages. */
bytes = 4096;
/* Align to page. */
offset &= 0xfffff000;
/* Kernel address of page where stall point stay. */
entryDump = (gctUINT8_PTR)entryDump + offset;
/* Align to page. */
gpuAddress &= 0xfffff000;
}
else
{
gcmkVERIFY_OK(gckOS_GPUPhysicalToCPUPhysical(os, gpuAddress, &cpuPhysical));
gcmkVERIFY_OK(gckOS_MapPhysical(os, (gctUINT32) cpuPhysical, 4096, &entry));
/* Align to page start. */
entryDump = (gctPOINTER)((gctUINTPTR_T)entry & ~0xFFF);
gpuAddress = gpuAddress & ~0xFFF;
bytes = 4096;
}
gcmkPRINT("User Command Buffer:\n");
_DumpBuffer(entryDump, gpuAddress, bytes);
if (kernel->virtualCommandBuffer)
{
gcmkVERIFY_OK(gckOS_DestroyKernelVirtualMapping(
os, buffer->virtualBuffer.physical, buffer->virtualBuffer.bytes, entry));
}
else
{
gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, 4096));
}
}
else
{
_DumpKernelCommandBuffer(Command);
}
/* Dump link queue. */
if (queue->count)
{
gcmkPRINT("Dump Level is %d, dump %d valid record in link queue:",
Command->kernel->stuckDump, queue->count);
dumpRear = queue->count;
for (i = 0; i < dumpRear; i++)
{
gckQUEUE_GetData(queue, i, &queueData);
linkData = &queueData->linkData;
/* Get gpu address of this command buffer. */
gpuAddress = linkData->start;
bytes = linkData->end - gpuAddress;
pid = linkData->pid;
gckOS_GetProcessNameByPid(pid, 16, processName);
if (kernel->virtualCommandBuffer)
{
buffer = gcvNULL;
/* Get the whole buffer. */
status = gckDEVICE_QueryGPUAddress(kernel->device, kernel, gpuAddress, &buffer);
if (gcmIS_ERROR(status))
{
/* Get kernel address of kernel command buffer. */
status = gckCOMMAND_AddressInKernelCommandBuffer(
kernel->command, gpuAddress, &entry);
if (gcmIS_ERROR(status))
{
status = gckHARDWARE_AddressInHardwareFuncions(
kernel->hardware, gpuAddress, &entry);
if (gcmIS_ERROR(status))
{
gcmkPRINT("Buffer [%08X - %08X] not found, may be freed",
linkData->start,
linkData->end);
continue;
}
}
offset = 0;
gcmkPRINT("Kernel Command Buffer: %08X, %08X", linkData->linkLow, linkData->linkHigh);
}
else
{
/* Get kernel logical for dump. */
if (buffer->virtualBuffer.kernelLogical)
{
/* Get kernel logical directly if it is a context buffer. */
entry = buffer->virtualBuffer.kernelLogical;
gcmkPRINT("Context Buffer: %08X, %08X PID:%d %s",
linkData->linkLow, linkData->linkHigh, linkData->pid, processName);
}
else
{
/* Make it accessiable by kernel if it is a user command buffer. */
gcmkVERIFY_OK(
gckOS_CreateKernelVirtualMapping(os,
buffer->virtualBuffer.physical,
buffer->virtualBuffer.bytes,
&entry,
&pageCount));
gcmkPRINT("User Command Buffer: %08X, %08X PID:%d %s",
linkData->linkLow, linkData->linkHigh, linkData->pid, processName);
}
offset = gpuAddress - buffer->virtualBuffer.gpuAddress;
}
/* Dump from the entry. */
_DumpBuffer((gctUINT8_PTR)entry + offset, gpuAddress, bytes);
/* Release kernel logical address if neccessary. */
if (buffer && !buffer->virtualBuffer.kernelLogical)
{
gcmkVERIFY_OK(
gckOS_DestroyKernelVirtualMapping(os,
buffer->virtualBuffer.physical,
buffer->virtualBuffer.bytes,
entry));
}
}
else
{
gcmkVERIFY_OK(gckOS_GPUPhysicalToCPUPhysical(os, gpuAddress, &cpuPhysical));
gcmkVERIFY_OK(gckOS_MapPhysical(os, (gctUINT32) cpuPhysical, bytes, &entry));
gcmkPRINT("Command Buffer: %08X, %08X PID:%d %s",
linkData->linkLow, linkData->linkHigh, linkData->pid, processName);
_DumpBuffer((gctUINT8_PTR)entry, gpuAddress, bytes);
gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, bytes));
}
}
}
return gcvSTATUS_OK;
}
gceSTATUS
gckCOMMAND_AddressInKernelCommandBuffer(
IN gckCOMMAND Command,
IN gctUINT32 Address,
OUT gctPOINTER * Pointer
)
{
gctINT i;
for (i = 0; i < gcdCOMMAND_QUEUES; i++)
{
if ((Address >= Command->queues[i].address)
&& (Address < (Command->queues[i].address + Command->pageSize))
)
{
*Pointer = (gctUINT8_PTR)Command->queues[i].logical
+ (Address - Command->queues[i].address)
;
return gcvSTATUS_OK;
}
}
return gcvSTATUS_NOT_FOUND;
}