/**
 * EMT worker function for DBGFR3OSDeregister.
 *
 * @returns VBox status code.
 * @param   pUVM    The user mode VM handle.
 * @param   pReg    The registration structure.
 */
static DECLCALLBACK(int) dbgfR3OSDeregister(PUVM pUVM, PDBGFOSREG pReg)
{
    /*
     * Unlink it.
     */
    bool    fWasCurOS = false;
    PDBGFOS pOSPrev   = NULL;
    PDBGFOS pOS;
    DBGF_OS_WRITE_LOCK(pUVM);
    for (pOS = pUVM->dbgf.s.pOSHead; pOS; pOSPrev = pOS, pOS = pOS->pNext)
        if (pOS->pReg == pReg)
        {
            if (pOSPrev)
                pOSPrev->pNext = pOS->pNext;
            else
                pUVM->dbgf.s.pOSHead = pOS->pNext;
            if (pUVM->dbgf.s.pCurOS == pOS)
            {
                pUVM->dbgf.s.pCurOS = NULL;
                fWasCurOS = true;
            }
            break;
        }
    DBGF_OS_WRITE_UNLOCK(pUVM);
    if (!pOS)
    {
        Log(("DBGFR3OSDeregister: %s -> VERR_NOT_FOUND\n", pReg->szName));
        return VERR_NOT_FOUND;
    }

    /*
     * Terminate it if it was the current OS, then invoke the
     * destructor and clean up.
     */
    if (fWasCurOS)
        pOS->pReg->pfnTerm(pUVM, pOS->abData);
    if (pOS->pReg->pfnDestruct)
        pOS->pReg->pfnDestruct(pUVM, pOS->abData);

    PDBGFOSEMTWRAPPER pFree = pOS->pWrapperHead;
    while ((pFree = pOS->pWrapperHead) != NULL)
    {
        pOS->pWrapperHead = pFree->pNext;
        pFree->pNext = NULL;
        MMR3HeapFree(pFree);
    }

    MMR3HeapFree(pOS);

    return VINF_SUCCESS;
}

/**
 * Create a User-kernel heap.
 *
 * This does not require SUPLib to be initialized as we'll lazily allocate the
 * kernel accessible memory on the first alloc call.
 *
 * @returns VBox status.
 * @param   pVM     The handle to the VM the heap should be associated with.
 * @param   ppHeap  Where to store the heap pointer.
 */
int mmR3UkHeapCreateU(PUVM pUVM, PMMUKHEAP *ppHeap)
{
    PMMUKHEAP pHeap = (PMMUKHEAP)MMR3HeapAllocZU(pUVM, MM_TAG_MM, sizeof(MMUKHEAP));
    if (pHeap)
    {
        int rc = RTCritSectInit(&pHeap->Lock);
        if (RT_SUCCESS(rc))
        {
            /*
             * Initialize the global stat record.
             */
            pHeap->pUVM = pUVM;
#ifdef MMUKHEAP_WITH_STATISTICS
            PMMUKHEAPSTAT pStat = &pHeap->Stat;
            STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.");
            STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.");
            STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cFrees",           STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.");
            STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
            STAMR3RegisterU(pUVM, &pStat->cbCurAllocated, sizeof(pStat->cbCurAllocated) == sizeof(uint32_t) ? STAMTYPE_U32 : STAMTYPE_U64,
                                                                        STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
            STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
            STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
#endif
            *ppHeap = pHeap;
            return VINF_SUCCESS;
        }
        AssertRC(rc);
        MMR3HeapFree(pHeap);
    }
    AssertMsgFailed(("failed to allocate heap structure\n"));
    return VERR_NO_MEMORY;
}

/**
 * Destroys a async I/O manager.
 *
 * @returns nothing.
 * @param   pAioMgr    The async I/O manager to destroy.
 */
static void pdmacFileAioMgrDestroy(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile, PPDMACEPFILEMGR pAioMgr)
{
    int rc = pdmacFileAioMgrShutdown(pAioMgr);
    AssertRC(rc);

    /* Unlink from the list. */
    rc = RTCritSectEnter(&pEpClassFile->CritSect);
    AssertRC(rc);

    PPDMACEPFILEMGR pPrev = pAioMgr->pPrev;
    PPDMACEPFILEMGR pNext = pAioMgr->pNext;

    if (pPrev)
        pPrev->pNext = pNext;
    else
        pEpClassFile->pAioMgrHead = pNext;

    if (pNext)
        pNext->pPrev = pPrev;

    pEpClassFile->cAioMgrs--;
    rc = RTCritSectLeave(&pEpClassFile->CritSect);
    AssertRC(rc);

    /* Free the resources. */
    RTCritSectDelete(&pAioMgr->CritSectBlockingEvent);
    RTSemEventDestroy(pAioMgr->EventSem);
    if (pAioMgr->enmMgrType != PDMACEPFILEMGRTYPE_SIMPLE)
        pdmacFileAioMgrNormalDestroy(pAioMgr);

    MMR3HeapFree(pAioMgr);
}

/**
 * Initializes the tracing.
 *
 * @returns VBox status code
 * @param   pVM                 The cross context VM structure.
 */
int dbgfR3TraceInit(PVM pVM)
{
    /*
     * Initialize the trace buffer handles.
     */
    Assert(NIL_RTTRACEBUF == (RTTRACEBUF)NULL);
    pVM->hTraceBufR3 = NIL_RTTRACEBUF;
    pVM->hTraceBufRC = NIL_RTRCPTR;
    pVM->hTraceBufR0 = NIL_RTR0PTR;

    /*
     * Check the config and enable tracing if requested.
     */
    PCFGMNODE pDbgfNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "DBGF");
#if defined(DEBUG) || defined(RTTRACE_ENABLED)
    bool const          fDefault        = false;
    const char * const  pszConfigDefault = "";
#else
    bool const          fDefault        = false;
    const char * const  pszConfigDefault = "";
#endif
    bool                fTracingEnabled;
    int rc = CFGMR3QueryBoolDef(pDbgfNode, "TracingEnabled", &fTracingEnabled, fDefault);
    AssertRCReturn(rc, rc);
    if (fTracingEnabled)
    {
        rc = dbgfR3TraceEnable(pVM, 0, 0);
        if (RT_SUCCESS(rc))
        {
            if (pDbgfNode)
            {
                char       *pszTracingConfig;
                rc = CFGMR3QueryStringAllocDef(pDbgfNode, "TracingConfig", &pszTracingConfig, pszConfigDefault);
                if (RT_SUCCESS(rc))
                {
                    rc = DBGFR3TraceConfig(pVM, pszTracingConfig);
                    if (RT_FAILURE(rc))
                        rc = VMSetError(pVM, rc, RT_SRC_POS, "TracingConfig=\"%s\" -> %Rrc", pszTracingConfig, rc);
                    MMR3HeapFree(pszTracingConfig);
                }
            }
            else
            {
                rc = DBGFR3TraceConfig(pVM, pszConfigDefault);
                if (RT_FAILURE(rc))
                    rc = VMSetError(pVM, rc, RT_SRC_POS, "TracingConfig=\"%s\" (default) -> %Rrc", pszConfigDefault, rc);
            }
        }
    }

    /*
     * Register a debug info item that will dump the trace buffer content.
     */
    if (RT_SUCCESS(rc))
        rc = DBGFR3InfoRegisterInternal(pVM, "tracebuf", "Display the trace buffer content. No arguments.", dbgfR3TraceInfo);

    return rc;
}

/**
 * Same as dbgfR3AsSearchEnv, except that the path is taken from the DBGF config
 * (CFGM).
 *
 * Nothing is done if the CFGM variable isn't set.
 *
 * @returns VBox status code.
 * @param   pszFilename     The filename.
 * @param   pszCfgValue     The name of the config variable (under /DBGF/).
 * @param   pfnOpen         The open callback function.
 * @param   pvUser          User argument for the callback.
 */
static int dbgfR3AsSearchCfgPath(PVM pVM, const char *pszFilename, const char *pszCfgValue, PFNDBGFR3ASSEARCHOPEN pfnOpen, void *pvUser)
{
    char *pszPath;
    int rc = CFGMR3QueryStringAllocDef(CFGMR3GetChild(CFGMR3GetRoot(pVM), "/DBGF"), pszCfgValue, &pszPath, NULL);
    if (RT_FAILURE(rc))
        return rc;
    if (!pszPath)
        return VERR_FILE_NOT_FOUND;
    rc = dbgfR3AsSearchPath(pszFilename, pszPath, pfnOpen, pvUser);
    MMR3HeapFree(pszPath);
    return rc;
}

/**
 * Terminate the network shaper.
 *
 * @returns VBox error code.
 * @param   pVM  The cross context VM structure.
 *
 * @remarks This method destroys all bandwidth group objects.
 */
int pdmR3NetShaperTerm(PVM pVM)
{
    PUVM pUVM = pVM->pUVM;
    AssertPtrReturn(pUVM, VERR_INVALID_POINTER);
    PPDMNETSHAPER pShaper = pUVM->pdm.s.pNetShaper;
    AssertPtrReturn(pShaper, VERR_INVALID_POINTER);

    /* Destroy the bandwidth managers. */
    PPDMNSBWGROUP pBwGroup = pShaper->pBwGroupsHead;
    while (pBwGroup)
    {
        PPDMNSBWGROUP pFree = pBwGroup;
        pBwGroup = pBwGroup->pNextR3;
        pdmNsBwGroupTerminate(pFree);
        MMR3HeapFree(pFree->pszNameR3);
        MMHyperFree(pVM, pFree);
    }

    RTCritSectDelete(&pShaper->Lock);
    MMR3HeapFree(pShaper);
    pUVM->pdm.s.pNetShaper = NULL;
    return VINF_SUCCESS;
}

/**
 * Delete an address space from the database.
 *
 * The address space must not be engaged as any of the standard aliases.
 *
 * @returns VBox status code.
 * @retval  VERR_SHARING_VIOLATION if in use as an alias.
 * @retval  VERR_NOT_FOUND if not found in the address space database.
 *
 * @param   pUVM        The user mode VM handle.
 * @param   hDbgAs      The address space handle. Aliases are not allowed.
 */
VMMR3DECL(int) DBGFR3AsDelete(PUVM pUVM, RTDBGAS hDbgAs)
{
    /*
     * Input validation. Retain the address space so it can be released outside
     * the lock as well as validated.
     */
    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
    if (hDbgAs == NIL_RTDBGAS)
        return VINF_SUCCESS;
    uint32_t cRefs = RTDbgAsRetain(hDbgAs);
    if (cRefs == UINT32_MAX)
        return VERR_INVALID_HANDLE;
    RTDbgAsRelease(hDbgAs);

    DBGF_AS_DB_LOCK_WRITE(pUVM);

    /*
     * You cannot delete any of the aliases.
     */
    for (size_t i = 0; i < RT_ELEMENTS(pUVM->dbgf.s.ahAsAliases); i++)
        if (pUVM->dbgf.s.ahAsAliases[i] == hDbgAs)
        {
            DBGF_AS_DB_UNLOCK_WRITE(pUVM);
            return VERR_SHARING_VIOLATION;
        }

    /*
     * Ok, try remove it from the database.
     */
    PDBGFASDBNODE pDbNode = (PDBGFASDBNODE)RTAvlPVRemove(&pUVM->dbgf.s.AsHandleTree, hDbgAs);
    if (!pDbNode)
    {
        DBGF_AS_DB_UNLOCK_WRITE(pUVM);
        return VERR_NOT_FOUND;
    }
    RTStrSpaceRemove(&pUVM->dbgf.s.AsNameSpace, pDbNode->NameCore.pszString);
    if (pDbNode->PidCore.Key != NIL_RTPROCESS)
        RTAvlU32Remove(&pUVM->dbgf.s.AsPidTree, pDbNode->PidCore.Key);

    DBGF_AS_DB_UNLOCK_WRITE(pUVM);

    /*
     * Free the resources.
     */
    RTDbgAsRelease(hDbgAs);
    MMR3HeapFree(pDbNode);

    return VINF_SUCCESS;
}

/**
 * Internal cleanup routine called by DBGFR3Term().
 *
 * @param   pUVM    The user mode VM handle.
 */
void dbgfR3OSTerm(PUVM pUVM)
{
    DBGF_OS_WRITE_LOCK(pUVM);

    /*
     * Terminate the current one.
     */
    if (pUVM->dbgf.s.pCurOS)
    {
        pUVM->dbgf.s.pCurOS->pReg->pfnTerm(pUVM, pUVM->dbgf.s.pCurOS->abData);
        pUVM->dbgf.s.pCurOS = NULL;
    }

    /*
     * Destroy all the instances.
     */
    while (pUVM->dbgf.s.pOSHead)
    {
        PDBGFOS pOS = pUVM->dbgf.s.pOSHead;
        pUVM->dbgf.s.pOSHead = pOS->pNext;
        if (pOS->pReg->pfnDestruct)
            pOS->pReg->pfnDestruct(pUVM, pOS->abData);

        PDBGFOSEMTWRAPPER pFree = pOS->pWrapperHead;
        while ((pFree = pOS->pWrapperHead) != NULL)
        {
            pOS->pWrapperHead = pFree->pNext;
            pFree->pNext = NULL;
            MMR3HeapFree(pFree);
        }

        MMR3HeapFree(pOS);
    }

    DBGF_OS_WRITE_UNLOCK(pUVM);
}

static void doTestsOnDefaultValues(PCFGMNODE pRoot)
{
    /* integer */
    uint64_t u64;
    RTTESTI_CHECK_RC(CFGMR3QueryU64(pRoot, "RamSize", &u64), VINF_SUCCESS);

    size_t cb = 0;
    RTTESTI_CHECK_RC(CFGMR3QuerySize(pRoot, "RamSize", &cb), VINF_SUCCESS);
    RTTESTI_CHECK(cb == sizeof(uint64_t));

    /* string */
    char *pszName = NULL;
    RTTESTI_CHECK_RC(CFGMR3QueryStringAlloc(pRoot, "Name", &pszName), VINF_SUCCESS);
    RTTESTI_CHECK_RC(CFGMR3QuerySize(pRoot, "Name", &cb), VINF_SUCCESS);
    RTTESTI_CHECK(cb == strlen(pszName) + 1);
    MMR3HeapFree(pszName);
}

/**
 * Ends a stack walk process.
 *
 * This *must* be called after a successful first call to any of the stack
 * walker functions. If not called we will leak memory or other resources.
 *
 * @param   pFirstFrame     The frame returned by one of the begin functions.
 */
VMMR3DECL(void) DBGFR3StackWalkEnd(PCDBGFSTACKFRAME pFirstFrame)
{
    if (    !pFirstFrame
        ||  !pFirstFrame->pFirstInternal)
        return;

    PDBGFSTACKFRAME pFrame = (PDBGFSTACKFRAME)pFirstFrame->pFirstInternal;
    while (pFrame)
    {
        PDBGFSTACKFRAME pCur = pFrame;
        pFrame = (PDBGFSTACKFRAME)pCur->pNextInternal;
        if (pFrame)
        {
            if (pCur->pSymReturnPC == pFrame->pSymPC)
                pFrame->pSymPC = NULL;
            if (pCur->pSymReturnPC == pFrame->pSymReturnPC)
                pFrame->pSymReturnPC = NULL;

            if (pCur->pSymPC == pFrame->pSymPC)
                pFrame->pSymPC = NULL;
            if (pCur->pSymPC == pFrame->pSymReturnPC)
                pFrame->pSymReturnPC = NULL;

            if (pCur->pLineReturnPC == pFrame->pLinePC)
                pFrame->pLinePC = NULL;
            if (pCur->pLineReturnPC == pFrame->pLineReturnPC)
                pFrame->pLineReturnPC = NULL;

            if (pCur->pLinePC == pFrame->pLinePC)
                pFrame->pLinePC = NULL;
            if (pCur->pLinePC == pFrame->pLineReturnPC)
                pFrame->pLineReturnPC = NULL;
        }

        RTDbgSymbolFree(pCur->pSymPC);
        RTDbgSymbolFree(pCur->pSymReturnPC);
        RTDbgLineFree(pCur->pLinePC);
        RTDbgLineFree(pCur->pLineReturnPC);

        pCur->pNextInternal = NULL;
        pCur->pFirstInternal = NULL;
        pCur->fFlags = 0;
        MMR3HeapFree(pCur);
    }
}

/**
 * Frees a task.
 *
 * @returns nothing.
 * @param   pEndpoint    Pointer to the endpoint the segment was for.
 * @param   pTask        The task to free.
 */
void pdmacFileTaskFree(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
{
    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;

    LogFlowFunc((": pEndpoint=%p pTask=%p\n", pEndpoint, pTask));

    /* Try the per endpoint cache first. */
    if (pEndpoint->cTasksCached < pEpClass->cTasksCacheMax)
    {
        /* Add it to the list. */
        pEndpoint->pTasksFreeTail->pNext = pTask;
        pEndpoint->pTasksFreeTail        = pTask;
        ASMAtomicIncU32(&pEndpoint->cTasksCached);
    }
    else
    {
        Log(("Freeing task %p because all caches are full\n", pTask));
        MMR3HeapFree(pTask);
    }
}

/**
 * Destruct a driver instance.
 *
 * Most VM resources are freed by the VM. This callback is provided so that any non-VM
 * resources can be freed correctly.
 *
 * @param   pDrvIns     The driver instance data.
 */
static DECLCALLBACK(void) drvscsihostDestruct(PPDMDRVINS pDrvIns)
{
    PDRVSCSIHOST pThis = PDMINS_2_DATA(pDrvIns, PDRVSCSIHOST);
    PDMDRV_CHECK_VERSIONS_RETURN_VOID(pDrvIns);

    RTFileClose(pThis->hDeviceFile);
    pThis->hDeviceFile = NIL_RTFILE;

    if (pThis->pszDevicePath)
    {
        MMR3HeapFree(pThis->pszDevicePath);
        pThis->pszDevicePath = NULL;
    }

    if (pThis->hQueueRequests != NIL_RTREQQUEUE)
    {
        int rc = RTReqQueueDestroy(pThis->hQueueRequests);
        AssertMsgRC(rc, ("Failed to destroy queue rc=%Rrc\n", rc));
        pThis->hQueueRequests = NIL_RTREQQUEUE;
    }

}

static int pdmacFileEpClose(PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
{
    int rc = VINF_SUCCESS;
    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->pEpClass;

    /* Make sure that all tasks finished for this endpoint. */
    rc = pdmacFileAioMgrCloseEndpoint(pEpFile->pAioMgr, pEpFile);
    AssertRC(rc);

    /*
     * If the async I/O manager is in failsafe mode this is the only endpoint
     * he processes and thus can be destroyed now.
     */
    if (pEpFile->pAioMgr->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
        pdmacFileAioMgrDestroy(pEpClassFile, pEpFile->pAioMgr);

    /* Free cached tasks. */
    PPDMACTASKFILE pTask = pEpFile->pTasksFreeHead;

    while (pTask)
    {
        PPDMACTASKFILE pTaskFree = pTask;
        pTask = pTask->pNext;
        MMR3HeapFree(pTaskFree);
    }

    /* Destroy the locked ranges tree now. */
    RTAvlrFileOffsetDestroy(pEpFile->AioMgr.pTreeRangesLocked, pdmacFileEpRangesLockedDestroy, NULL);

    RTFileClose(pEpFile->hFile);

#ifdef VBOX_WITH_STATISTICS
    STAMR3Deregister(pEpClassFile->Core.pVM, &pEpFile->StatRead);
    STAMR3Deregister(pEpClassFile->Core.pVM, &pEpFile->StatWrite);
#endif

    return VINF_SUCCESS;
}

static int pdmacFileEpClose(PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
{
    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile      = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
    PPDMASYNCCOMPLETIONEPCLASSFILE  pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->pEpClass;

    /* Make sure that all tasks finished for this endpoint. */
    int rc = pdmacFileAioMgrCloseEndpoint(pEpFile->pAioMgr, pEpFile);
    AssertRC(rc);

    /*
     * If the async I/O manager is in failsafe mode this is the only endpoint
     * he processes and thus can be destroyed now.
     */
    if (pEpFile->pAioMgr->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
        pdmacFileAioMgrDestroy(pEpClassFile, pEpFile->pAioMgr);

    /* Free cached tasks. */
    PPDMACTASKFILE pTask = pEpFile->pTasksFreeHead;

    while (pTask)
    {
        PPDMACTASKFILE pTaskFree = pTask;
        pTask = pTask->pNext;
        MMR3HeapFree(pTaskFree);
    }

    /* Destroy the locked ranges tree now. */
    RTAvlrFileOffsetDestroy(pEpFile->AioMgr.pTreeRangesLocked, pdmacFileEpRangesLockedDestroy, NULL);

    RTFileClose(pEpFile->hFile);

#ifdef VBOX_WITH_STATISTICS
    /* Not sure if this might be unnecessary because of similar statement in pdmR3AsyncCompletionStatisticsDeregister? */
    STAMR3DeregisterF(pEpClassFile->Core.pVM->pUVM, "/PDM/AsyncCompletion/File/%s/*", RTPathFilename(pEpFile->Core.pszUri));
#endif

    return VINF_SUCCESS;
}

/**
 * Internal cleanup routine called by DBGFR3Term().
 *
 * @param   pVM     Pointer to the shared VM structure.
 */
void dbgfR3OSTerm(PVM pVM)
{
    /*
     * Terminate the current one.
     */
    if (pVM->dbgf.s.pCurOS)
    {
        pVM->dbgf.s.pCurOS->pReg->pfnTerm(pVM, pVM->dbgf.s.pCurOS->abData);
        pVM->dbgf.s.pCurOS = NULL;
    }

    /*
     * Destroy all the instances.
     */
    while (pVM->dbgf.s.pOSHead)
    {
        PDBGFOS pOS = pVM->dbgf.s.pOSHead;
        pVM->dbgf.s.pOSHead = pOS->pNext;
        if (pOS->pReg->pfnDestruct)
            pOS->pReg->pfnDestruct(pVM, pOS->abData);
        MMR3HeapFree(pOS);
    }
}

/***
 * Worker for mmR3UkHeapAlloc that creates and adds a new sub-heap.
 *
 * @returns Pointer to the new sub-heap.
 * @param   pHeap       The heap
 * @param   cbSubHeap   The size of the sub-heap.
 */
static PMMUKHEAPSUB mmR3UkHeapAddSubHeap(PMMUKHEAP pHeap, size_t cbSubHeap)
{
    PMMUKHEAPSUB pSubHeap = (PMMUKHEAPSUB)MMR3HeapAllocU(pHeap->pUVM, MM_TAG_MM/*_UK_HEAP*/, sizeof(*pSubHeap));
    if (pSubHeap)
    {
        pSubHeap->cb = cbSubHeap;
        int rc = SUPR3PageAllocEx(pSubHeap->cb >> PAGE_SHIFT, 0, &pSubHeap->pv, &pSubHeap->pvR0, NULL);
        if (RT_SUCCESS(rc))
        {
            rc = RTHeapSimpleInit(&pSubHeap->hSimple, pSubHeap->pv, pSubHeap->cb);
            if (RT_SUCCESS(rc))
            {
                pSubHeap->pNext = pHeap->pSubHeapHead;
                pHeap->pSubHeapHead = pSubHeap;
                return pSubHeap;
            }

            /* bail out */
            SUPR3PageFreeEx(pSubHeap->pv, pSubHeap->cb >> PAGE_SHIFT);
        }
        MMR3HeapFree(pSubHeap);
    }
    return NULL;
}

/**
 * Initialize the network shaper.
 *
 * @returns VBox status code
 * @param   pVM Pointer to the VM.
 */
int pdmR3NetShaperInit(PVM pVM)
{
    LogFlowFunc((": pVM=%p\n", pVM));

    VM_ASSERT_EMT(pVM);

    PPDMNETSHAPER pNetShaper = NULL;

    int rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_NET_SHAPER,
                              sizeof(PDMNETSHAPER),
                              (void **)&pNetShaper);
    if (RT_SUCCESS(rc))
    {
        PCFGMNODE pCfgRoot      = CFGMR3GetRoot(pVM);
        PCFGMNODE pCfgNetShaper = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "NetworkShaper");

        pNetShaper->pVM = pVM;
        rc = RTCritSectInit(&pNetShaper->cs);
        if (RT_SUCCESS(rc))
        {
            /* Create all bandwidth groups. */
            PCFGMNODE pCfgBwGrp = CFGMR3GetChild(pCfgNetShaper, "BwGroups");

            if (pCfgBwGrp)
            {
                for (PCFGMNODE pCur = CFGMR3GetFirstChild(pCfgBwGrp); pCur; pCur = CFGMR3GetNextChild(pCur))
                {
                    uint64_t cbMax;
                    size_t cbName = CFGMR3GetNameLen(pCur) + 1;
                    char *pszBwGrpId = (char *)RTMemAllocZ(cbName);

                    if (!pszBwGrpId)
                    {
                        rc = VERR_NO_MEMORY;
                        break;
                    }

                    rc = CFGMR3GetName(pCur, pszBwGrpId, cbName);
                    AssertRC(rc);

                    if (RT_SUCCESS(rc))
                        rc = CFGMR3QueryU64(pCur, "Max", &cbMax);
                    if (RT_SUCCESS(rc))
                        rc = pdmNsBwGroupCreate(pNetShaper, pszBwGrpId, cbMax);

                    RTMemFree(pszBwGrpId);

                    if (RT_FAILURE(rc))
                        break;
                }
            }

            if (RT_SUCCESS(rc))
            {
                PUVM pUVM = pVM->pUVM;
                AssertMsg(!pUVM->pdm.s.pNetShaper, ("Network shaper was already initialized\n"));

                char szDesc[64];
                static unsigned s_iThread;

                RTStrPrintf(szDesc, sizeof(szDesc), "PDMNsTx-%d", ++s_iThread);
                rc = PDMR3ThreadCreate(pVM, &pNetShaper->hTxThread, pNetShaper,
                                       pdmR3NsTxThread, pdmR3NsTxWakeUp, 0,
                                       RTTHREADTYPE_IO, szDesc);
                if (RT_SUCCESS(rc))
                {
                    pUVM->pdm.s.pNetShaper = pNetShaper;
                    return VINF_SUCCESS;
                }
            }

            RTCritSectDelete(&pNetShaper->cs);
        }
        MMR3HeapFree(pNetShaper);
    }

    LogFlowFunc((": pVM=%p rc=%Rrc\n", pVM, rc));
    return rc;
}

/**
 * Reallocate memory allocated with MMR3HeapAlloc() or MMR3HeapRealloc().
 *
 * @returns Pointer to reallocated memory.
 * @param   pv          Pointer to the memory block to reallocate.
 *                      Must not be NULL!
 * @param   cbNewSize   New block size.
 */
VMMR3DECL(void *) MMR3HeapRealloc(void *pv, size_t cbNewSize)
{
    AssertMsg(pv, ("Invalid pointer pv=%p\n", pv));
    if (!pv)
        return NULL;

    /*
     * If newsize is zero then this is a free.
     */
    if (!cbNewSize)
    {
        MMR3HeapFree(pv);
        return NULL;
    }

    /*
     * Validate header.
     */
    PMMHEAPHDR  pHdr = (PMMHEAPHDR)pv - 1;
    if (    pHdr->cbSize & (MMR3HEAP_SIZE_ALIGNMENT - 1)
        ||  (uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1))
    {
        AssertMsgFailed(("Invalid heap header! pv=%p, size=%#x\n", pv, pHdr->cbSize));
        return NULL;
    }
    Assert(pHdr->pStat != NULL);
    Assert(!((uintptr_t)pHdr->pNext & (RTMEM_ALIGNMENT - 1)));
    Assert(!((uintptr_t)pHdr->pPrev & (RTMEM_ALIGNMENT - 1)));

    PMMHEAP pHeap = pHdr->pStat->pHeap;

#ifdef MMR3HEAP_WITH_STATISTICS
    RTCritSectEnter(&pHeap->Lock);
    pHdr->pStat->cReallocations++;
    pHeap->Stat.cReallocations++;
    RTCritSectLeave(&pHeap->Lock);
#endif

    /*
     * Reallocate the block.
     */
    cbNewSize = RT_ALIGN_Z(cbNewSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
    PMMHEAPHDR pHdrNew = (PMMHEAPHDR)RTMemRealloc(pHdr, cbNewSize);
    if (!pHdrNew)
    {
#ifdef MMR3HEAP_WITH_STATISTICS
        RTCritSectEnter(&pHeap->Lock);
        pHdr->pStat->cFailures++;
        pHeap->Stat.cFailures++;
        RTCritSectLeave(&pHeap->Lock);
#endif
        return NULL;
    }

    /*
     * Update pointers.
     */
    if (pHdrNew != pHdr)
    {
        RTCritSectEnter(&pHeap->Lock);
        if (pHdrNew->pPrev)
            pHdrNew->pPrev->pNext = pHdrNew;
        else
            pHeap->pHead = pHdrNew;

        if (pHdrNew->pNext)
            pHdrNew->pNext->pPrev = pHdrNew;
        else
            pHeap->pTail = pHdrNew;
        RTCritSectLeave(&pHeap->Lock);
    }

    /*
     * Update statistics.
     */
#ifdef MMR3HEAP_WITH_STATISTICS
    RTCritSectEnter(&pHeap->Lock);
    pHdrNew->pStat->cbAllocated += cbNewSize - pHdrNew->cbSize;
    pHeap->Stat.cbAllocated += cbNewSize - pHdrNew->cbSize;
    RTCritSectLeave(&pHeap->Lock);
#endif

    pHdrNew->cbSize = cbNewSize;

    return pHdrNew + 1;
}

//.........这里部分代码省略.........

    /* IBase. */
    pDrvIns->IBase.pfnQueryInterface     = drvdiskintQueryInterface;

    /* IMedia */
    pThis->IMedia.pfnRead                = drvdiskintRead;
    pThis->IMedia.pfnWrite               = drvdiskintWrite;
    pThis->IMedia.pfnFlush               = drvdiskintFlush;
    pThis->IMedia.pfnGetSize             = drvdiskintGetSize;
    pThis->IMedia.pfnIsReadOnly          = drvdiskintIsReadOnly;
    pThis->IMedia.pfnBiosGetPCHSGeometry = drvdiskintBiosGetPCHSGeometry;
    pThis->IMedia.pfnBiosSetPCHSGeometry = drvdiskintBiosSetPCHSGeometry;
    pThis->IMedia.pfnBiosGetLCHSGeometry = drvdiskintBiosGetLCHSGeometry;
    pThis->IMedia.pfnBiosSetLCHSGeometry = drvdiskintBiosSetLCHSGeometry;
    pThis->IMedia.pfnGetUuid             = drvdiskintGetUuid;

    /* IMediaAsync */
    pThis->IMediaAsync.pfnStartRead      = drvdiskintStartRead;
    pThis->IMediaAsync.pfnStartWrite     = drvdiskintStartWrite;
    pThis->IMediaAsync.pfnStartFlush     = drvdiskintStartFlush;

    /* IMediaAsyncPort. */
    pThis->IMediaAsyncPort.pfnTransferCompleteNotify  = drvdiskintAsyncTransferCompleteNotify;

    /* IMediaPort. */
    pThis->IMediaPort.pfnQueryDeviceLocation = drvdiskintQueryDeviceLocation;

    /* Query the media port interface above us. */
    pThis->pDrvMediaPort = PDMIBASE_QUERY_INTERFACE(pDrvIns->pUpBase, PDMIMEDIAPORT);
    if (!pThis->pDrvMediaPort)
        return PDMDRV_SET_ERROR(pDrvIns, VERR_PDM_MISSING_INTERFACE_BELOW,
                                N_("No media port inrerface above"));

    /* Try to attach async media port interface above.*/
    pThis->pDrvMediaAsyncPort = PDMIBASE_QUERY_INTERFACE(pDrvIns->pUpBase, PDMIMEDIAASYNCPORT);

    /*
     * Try attach driver below and query it's media interface.
     */
    PPDMIBASE pBase;
    rc = PDMDrvHlpAttach(pDrvIns, fFlags, &pBase);
    if (RT_FAILURE(rc))
        return PDMDrvHlpVMSetError(pDrvIns, rc, RT_SRC_POS,
                                   N_("Failed to attach driver below us! %Rrc"), rc);

    pThis->pDrvMedia = PDMIBASE_QUERY_INTERFACE(pBase, PDMIMEDIA);
    if (!pThis->pDrvMedia)
        return PDMDRV_SET_ERROR(pDrvIns, VERR_PDM_MISSING_INTERFACE_BELOW,
                                N_("No media or async media interface below"));

    pThis->pDrvMediaAsync = PDMIBASE_QUERY_INTERFACE(pBase, PDMIMEDIAASYNC);

    if (pThis->pDrvMedia->pfnDiscard)
        pThis->IMedia.pfnDiscard = drvdiskintDiscard;
    if (   pThis->pDrvMediaAsync
        && pThis->pDrvMediaAsync->pfnStartDiscard)
        pThis->IMediaAsync.pfnStartDiscard = drvdiskintStartDiscard;

    if (pThis->fCheckConsistency)
    {
        /* Create the AVL tree. */
        pThis->pTreeSegments = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
        if (!pThis->pTreeSegments)
            rc = VERR_NO_MEMORY;
    }

    if (pThis->fTraceRequests)
    {
        for (unsigned i = 0; i < RT_ELEMENTS(pThis->apReqActive); i++)
        {
            pThis->apReqActive[i].pIoReq  = NULL;
            pThis->apReqActive[i].tsStart = 0;
        }

        pThis->iNextFreeSlot = 0;

        /* Init event semaphore. */
        rc = RTSemEventCreate(&pThis->SemEvent);
        AssertRC(rc);
        pThis->fRunning = true;
        rc = RTThreadCreate(&pThis->hThread, drvdiskIntIoReqExpiredCheck, pThis,
                            0, RTTHREADTYPE_INFREQUENT_POLLER, 0, "DiskIntegrity");
        AssertRC(rc);
    }

    if (pThis->fCheckDoubleCompletion)
    {
        pThis->iEntry = 0;
        pThis->papIoReq = (PDRVDISKAIOREQ *)RTMemAllocZ(pThis->cEntries * sizeof(PDRVDISKAIOREQ));
        AssertPtr(pThis->papIoReq);
    }

    if (pszIoLogFilename)
    {
        rc = VDDbgIoLogCreate(&pThis->hIoLogger, pszIoLogFilename, VDDBG_IOLOG_LOG_DATA);
        MMR3HeapFree(pszIoLogFilename);
    }

    return rc;
}

//.........这里部分代码省略.........

                pEpFile->pTasksFreeTail = pEpFile->pTasksFreeHead;
                pEpFile->cTasksCached   = 0;
                pEpFile->enmBackendType = enmEpBackend;
                /*
                 * Disable async flushes on Solaris for now.
                 * They cause weird hangs which needs more investigations.
                 */
#ifndef RT_OS_SOLARIS
                pEpFile->fAsyncFlushSupported = true;
#else
                pEpFile->fAsyncFlushSupported = false;
#endif

                if (enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
                {
                    /* Simple mode. Every file has its own async I/O manager. */
                    rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgr, PDMACEPFILEMGRTYPE_SIMPLE);
                }
                else
                {
                    pAioMgr = pEpClassFile->pAioMgrHead;

                    /* Check for an idling manager of the same type */
                    while (pAioMgr)
                    {
                        if (pAioMgr->enmMgrType == enmMgrType)
                            break;
                        pAioMgr = pAioMgr->pNext;
                    }

                    if (!pAioMgr)
                        rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgr, enmMgrType);
                }

                if (RT_SUCCESS(rc))
                {
                    pEpFile->AioMgr.pTreeRangesLocked = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
                    if (!pEpFile->AioMgr.pTreeRangesLocked)
                        rc = VERR_NO_MEMORY;
                    else
                    {
                        pEpFile->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;

                        /* Assign the endpoint to the thread. */
                        rc = pdmacFileAioMgrAddEndpoint(pAioMgr, pEpFile);
                        if (RT_FAILURE(rc))
                        {
                            RTMemFree(pEpFile->AioMgr.pTreeRangesLocked);
                            MMR3HeapFree(pEpFile->pTasksFreeHead);
                        }
                    }
                }
                else if (rc == VERR_FILE_AIO_INSUFFICIENT_EVENTS)
                {
                    PUVM pUVM = VMR3GetUVM(pEpClassFile->Core.pVM);
#if defined(RT_OS_LINUX)
                    rc = VMR3SetError(pUVM, rc, RT_SRC_POS,
                                      N_("Failed to create I/O manager for VM due to insufficient resources on the host. "
                                         "Either increase the amount of allowed events in /proc/sys/fs/aio-max-nr or enable "
                                         "the host I/O cache"));
#else
                    rc = VMR3SetError(pUVM, rc, RT_SRC_POS,
                                      N_("Failed to create I/O manager for VM due to insufficient resources on the host. "
                                         "Enable the host I/O cache"));
#endif
                }
                else
                {
                    PUVM pUVM = VMR3GetUVM(pEpClassFile->Core.pVM);
                    rc = VMR3SetError(pUVM, rc, RT_SRC_POS,
                                      N_("Failed to create I/O manager for VM due to an unknown error"));
                }
            }
        }

        if (RT_FAILURE(rc))
            RTFileClose(pEpFile->hFile);
    }

#ifdef VBOX_WITH_STATISTICS
    if (RT_SUCCESS(rc))
    {
        STAMR3RegisterF(pEpClassFile->Core.pVM, &pEpFile->StatRead,
                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
                       STAMUNIT_TICKS_PER_CALL, "Time taken to read from the endpoint",
                       "/PDM/AsyncCompletion/File/%s/Read", RTPathFilename(pEpFile->Core.pszUri));

        STAMR3RegisterF(pEpClassFile->Core.pVM, &pEpFile->StatWrite,
                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
                       STAMUNIT_TICKS_PER_CALL, "Time taken to write to the endpoint",
                       "/PDM/AsyncCompletion/File/%s/Write", RTPathFilename(pEpFile->Core.pszUri));
    }
#endif

    if (RT_SUCCESS(rc))
        LogRel(("AIOMgr: Endpoint for file '%s' (flags %08x) created successfully\n", pszUri, pEpFile->fFlags));

    return rc;
}