private static async Task<ApplicationManager> CreateApplicationInternal()
        {
            int siteIndex;

            // try till succeeded
            while (!_availableSiteIndex.TryPop(out siteIndex))
            {
                await Task.Delay(5000);
            }

            string applicationName = _sitePrefix + siteIndex;

            string operationName = "SitePool.CreateApplicationInternal " + applicationName;
            
            var siteManager = GetSiteManager(new KuduTestContext());

            Site site = siteManager.GetSite(applicationName);
            if (site != null)
            {
                TestTracer.Trace("{0} Site already exists at {1}. Reusing site", operationName, site.SiteUrl);
                var appManager = new ApplicationManager(siteManager, site, applicationName)
                {
                    SitePoolIndex = siteIndex
                };

                // In site reuse mode, clean out the existing site so we start clean
                // Enumrate all w3wp processes and make sure to kill any process with an open handle to klr.host.dll
                foreach (var process in (await appManager.ProcessManager.GetProcessesAsync()).Where(p => p.Name.Equals("w3wp", StringComparison.OrdinalIgnoreCase)))
                {
                    var extendedProcess = await appManager.ProcessManager.GetProcessAsync(process.Id);
                    if (extendedProcess.OpenFileHandles.Any(h => h.IndexOf("dnx.host.dll", StringComparison.OrdinalIgnoreCase) != -1))
                    {
                        await appManager.ProcessManager.KillProcessAsync(extendedProcess.Id, throwOnError:false);
                    }
                }

                await appManager.RepositoryManager.Delete(deleteWebRoot: true, ignoreErrors: true);

                // Make sure we start with the correct default file as some tests expect it
                WriteIndexHtml(appManager);

                TestTracer.Trace("{0} completed", operationName);
                return appManager;
            }
            else
            {
                TestTracer.Trace("{0} Creating new site", operationName);
                site = await siteManager.CreateSiteAsync(applicationName);

                TestTracer.Trace("{0} Created new site at {1}", operationName, site.SiteUrl);
                return new ApplicationManager(siteManager, site, applicationName)
                {
                    SitePoolIndex = siteIndex
                };
            }
        }

 public static void ReportTestCompletion(ApplicationManager applicationManager, bool success)
 {
     if (success || _availableSiteIndex.Count <= 1)
     {
         _availableSiteIndex.Push(applicationManager.SitePoolIndex);
     }
     else
     {
         TestTracer.Trace("SitePool.ReportTestCompletion Removing application {0} from pool", applicationManager.ApplicationName);
     }
 }

        private void VerifyCommand(CommandTestSettings commandTestSettings, ApplicationManager appManager)
        {
            TestTracer.Trace("Running command - '{0}' on '{1}'", commandTestSettings.Command, commandTestSettings.WorkingDirectory);
            CommandResult commandResult = appManager.CommandExecutor.ExecuteCommand(commandTestSettings.Command, commandTestSettings.WorkingDirectory).Result;

            TestTracer.Trace("Received result\nOutput\n======\n{0}\nError\n======\n{1}\nExit Code - {2}", commandResult.Output, commandResult.Error, commandResult.ExitCode);

            Assert.Equal(commandTestSettings.ExpectedResult.ExitCode, commandResult.ExitCode);
            AssertOutput(commandTestSettings.ExpectedResult.Error, commandResult.Error);
            AssertOutput(commandTestSettings.ExpectedResult.Output, commandResult.Output);
        }

        public static void TraceDeploymentLog(ApplicationManager appManager, string id)
        {
            Trace("\n====================================================================================\n\t\tDeployment Log for " + id + "\n=====================================================================================");

            var entries = appManager.DeploymentManager.GetLogEntriesAsync(id).Result.ToList();
            var allDetails = entries.Where(e => e.DetailsUrl != null)
                                    .SelectMany(e => appManager.DeploymentManager.GetLogEntryDetailsAsync(id, e.Id).Result).ToList();
            var allEntries = entries.Concat(allDetails).ToList();
            foreach (var entry in allEntries)
            {
                Trace(entry.LogTime, entry.Message);
            }
        }

        public static async Task<ApplicationManager> CreateApplicationAsync()
        {
            await _semaphore.WaitAsync();
            ApplicationManager appManager = _nextAppManager;
            _nextAppManager = null;
            _semaphore.Release();

            if (appManager == null)
            {
                appManager = await CreateApplicationInternal();
            }

            EnsureNextApplication();
            return appManager;
        }

        private static async Task<ApplicationManager> CreateApplicationInternal()
        {
            int siteIndex;
            _availableSiteIndex.TryPop(out siteIndex);
            string applicationName = _sitePrefix + siteIndex;

            string operationName = "SitePool.CreateApplicationInternal " + applicationName;
            
            var pathResolver = new DefaultPathResolver(PathHelper.ServiceSitePath, PathHelper.SitesPath);
            var settingsResolver = new DefaultSettingsResolver();

            var siteManager = GetSiteManager(pathResolver, settingsResolver);

            Site site = siteManager.GetSite(applicationName);
            if (site != null)
            {
                TestTracer.Trace("{0} Site already exists at {1}. Reusing site", operationName, site.SiteUrl);
                var appManager = new ApplicationManager(siteManager, site, applicationName, settingsResolver)
                {
                    SitePoolIndex = siteIndex
                };

                // In site reuse mode, clean out the existing site so we start clean
                await appManager.RepositoryManager.Delete(deleteWebRoot: true, ignoreErrors: true);

                // Make sure we start with the correct default file as some tests expect it
                WriteIndexHtml(appManager);

                TestTracer.Trace("{0} completed", operationName);
                return appManager;
            }
            else
            {
                TestTracer.Trace("{0} Creating new site", operationName);
                site = await siteManager.CreateSiteAsync(applicationName);

                TestTracer.Trace("{0} Created new site at {1}", operationName, site.SiteUrl);
                return new ApplicationManager(siteManager, site, applicationName, settingsResolver)
                {
                    SitePoolIndex = siteIndex
                };
            }
        }

        private void DoGitRedeploy(ApplicationManager appManager, string commitId, string verificationContent)
        {
            DeployResult result = null;
            try
            {
                appManager.DeploymentManager.DeployAsync(commitId).Wait();
                var results = appManager.DeploymentManager.GetResultsAsync().Result.ToList();
                result = results.Where(r => r.Id == commitId).FirstOrDefault();
            }
            catch(Exception ex)
            {
                string msg = string.Format("Redeploy operation failed for commit ID:  {0}.  Exception:  {1}", commitId, ex.ToString() );
                throw new ApplicationException(msg) ;
            }

            if (result == null)
            {
                string msg = string.Format("Redeploy operation completed but expected commit Id was not deployed.  Commit ID:  {0}.", commitId);
                throw new ApplicationException(msg);
            }

            StressUtils.VerifySite(appManager.SiteUrl, verificationContent);
        }

        private void PushAndVerifyConsoleWorker(ApplicationManager appManager, TestRepository testRepository, string[] expectedVerificationFileLines, int expectedDeployments = 1)
        {
            appManager.GitDeploy(testRepository.PhysicalPath);
            var results = appManager.DeploymentManager.GetResultsAsync().Result.ToList();

            Assert.Equal(expectedDeployments, results.Count);
            for (int i = 0; i < expectedDeployments; i++)
            {
                Assert.Equal(DeployStatus.Success, results[i].Status);
            }

            var expectedContinuousJob = new ContinuousJob()
            {
                Name = "deployedJob",
                JobType = "continuous",
                Status = "Running",
                RunCommand = "ConsoleWorker.exe"
            };

            WaitUntilAssertVerified(
                "verify continuous job",
                TimeSpan.FromSeconds(60),
                () =>
                {
                    ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("deployedJob").Result;
                    AssertContinuousJob(expectedContinuousJob, deployedJob);
                });

            WaitUntilAssertVerified(
                "verification file",
                TimeSpan.FromSeconds(30),
                () =>
                {
                    VerifyVerificationFile(appManager, expectedVerificationFileLines);
                });
        }

        private async Task AssertUrlContentAsync(ApplicationManager appManager, Uri requestUrl, string expectedContent)
        {
            if (expectedContent == null)
            {
                Assert.Null(requestUrl);
                return;
            }

            string address = requestUrl.ToString();
            using (var httpClient = HttpClientHelper.CreateClient(address, appManager.JobsManager.Credentials))
            {
                using (var response = await httpClient.GetAsync(String.Empty))
                {
                    var content = await response.Content.ReadAsStringAsync();
                    TestTracer.Trace("Request to: {0}\nStatus code: {1}\nContent: {2}", address, response.StatusCode, content);

                    Assert.True(content.IndexOf(expectedContent, StringComparison.OrdinalIgnoreCase) >= 0, "Expected content: " + expectedContent);
                }
            }
        }

        private void VerifyVerificationFile(ApplicationManager appManager, string[] expectedContentLines)
        {
            var logFiles = appManager.VfsManager.ListAsync("LogFiles").Result;
            string[] verificationFileNames = logFiles.Where(logFile => logFile.Name.StartsWith("verification.txt", StringComparison.OrdinalIgnoreCase)).Select(logFile => logFile.Name).ToArray();

            for (int fileIndex = 0; fileIndex < verificationFileNames.Length; fileIndex++)
            {
                // Make sure at least one file is verified
                try
                {
                    string verificationFileName = verificationFileNames[fileIndex];
                    string verificationFileContent = appManager.VfsManager.ReadAllText("LogFiles/" + verificationFileName).TrimEnd();
                    Assert.NotNull(verificationFileContent);
                    string[] lines = verificationFileContent.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
                    Assert.Equal(expectedContentLines.Length, lines.Length);
                    for (int i = 0; i < expectedContentLines.Length; i++)
                    {
                        Assert.Equal(expectedContentLines[i], lines[i].Trim());
                    }

                    return;
                }
                catch
                {
                    if (fileIndex == verificationFileNames.Length - 1)
                    {
                        throw;
                    }
                }
            }
        }

 private static void RestartServiceSite(ApplicationManager appManager)
 {
     try
     {
         appManager.ProcessManager.KillProcessAsync(0).Wait();
     }
     catch
     {
     }
 }

        private void VerifyTriggeredJobTriggers(ApplicationManager appManager, string jobName, int expectedNumberOfRuns, string expectedStatus, string expectedOutput = null, string expectedError = null, string arguments = null, bool scheduledTriggeredJob = false)
        {
            if (!scheduledTriggeredJob)
            {
                appManager.JobsManager.InvokeTriggeredJobAsync(jobName, arguments).Wait();
            }

            try
            {
                WaitUntilAssertVerified(
                    "verify triggered job run",
                    TimeSpan.FromSeconds(30),
                    () =>
                    {
                        TriggeredJobHistory triggeredJobHistory = appManager.JobsManager.GetTriggeredJobHistoryAsync(jobName).Result;
                        Assert.NotNull(triggeredJobHistory);
                        Assert.Equal(expectedNumberOfRuns, triggeredJobHistory.TriggeredJobRuns.Count());

                        TriggeredJobRun triggeredJobRun = triggeredJobHistory.TriggeredJobRuns.FirstOrDefault();
                        AssertTriggeredJobRun(appManager, triggeredJobRun, jobName, expectedStatus, expectedOutput, expectedError);
                    });
            }
            catch
            {
                // On error trace the scheduler log if it is a scheduler job
                if (scheduledTriggeredJob)
                {
                    try
                    {
                        string schedulerLog = appManager.VfsManager.ReadAllText(JobsDataPath + "/triggered/" + jobName + "/job_scheduler.log");
                        TestTracer.Trace("Scheduler log - " + schedulerLog);
                    }
                    catch
                    {
                    }
                }

                throw;
            }
        }

 private void CleanupTest(ApplicationManager appManager)
 {
     WaitUntilAssertVerified(
         "clean site for jobs",
         TimeSpan.FromSeconds(60),
         () =>
         {
             appManager.VfsManager.Delete(JobsBinPath, recursive: true);
             appManager.VfsManager.Delete(JobsDataPath, recursive: true);
             appManager.VfsManager.Delete(VerificationFilePath);
         });
 }

        private void VerifyTriggeredJobTriggers(ApplicationManager appManager, string jobName, int expectedNumberOfRuns, string expectedStatus, string expectedOutput = null, string expectedError = null)
        {
            appManager.JobsManager.InvokeTriggeredJobAsync(jobName).Wait();

            WaitUntilAssertVerified(
                "verify triggered job run",
                TimeSpan.FromSeconds(20),
                () =>
                {
                    TriggeredJobHistory triggeredJobHistory = appManager.JobsManager.GetTriggeredJobHistoryAsync(jobName).Result;
                    Assert.NotNull(triggeredJobHistory);
                    Assert.Equal(expectedNumberOfRuns, triggeredJobHistory.TriggeredJobRuns.Count());

                    foreach (TriggeredJobRun triggeredJobRun in triggeredJobHistory.TriggeredJobRuns)
                    {
                        AssertTriggeredJobRun(appManager, triggeredJobRun, expectedStatus, expectedOutput, expectedError);
                    }
                });
        }

        private void DoGitPush(ApplicationManager appManager, TestRepository testRepository, string appName, string verificationContent)
        {
            appManager.GitDeploy(testRepository.PhysicalPath, "master");

            var results = appManager.DeploymentManager.GetResultsAsync().Result.ToList();
            if (results[0].Status != Kudu.Core.Deployment.DeployStatus.Success)
            {
                string msg = string.Format("Deployment of app {0} failed.  Deployment count:  {1} .   Deployment Status:  {2}", appName, results.Count, results[0].Status);
                throw new ApplicationException(msg);
            }
            StressUtils.VerifySite(appManager.SiteUrl, verificationContent);
        }

 private void VerifyVerificationFile(ApplicationManager appManager, string[] expectedContentLines)
 {
     string verificationFileContent = appManager.VfsManager.ReadAllText(VerificationFilePath).TrimEnd();
     Assert.NotNull(verificationFileContent);
     string[] lines = verificationFileContent.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
     Assert.Equal(expectedContentLines.Length, lines.Length);
     for (int i = 0; i < expectedContentLines.Length; i++)
     {
         Assert.Equal(expectedContentLines[i], lines[i].Trim());
     }
 }

 public static void ReportTestCompletion(ApplicationManager applicationManager, bool success)
 {
     _availableSiteIndex.Push(applicationManager.SitePoolIndex);
 }

 private static void WaitForRepositorySite(ApplicationManager appManager)
 {
     HttpUtils.WaitForSite(appManager.ServiceUrl);
 }

        private static async Task<ApplicationManager> CreateApplicationInternal(int siteIndex)
        {
            string applicationName = _sitePrefix + siteIndex;

            string operationName = "SitePool.CreateApplicationInternal " + applicationName;

            var context = new KuduTestContext();
            var siteManager = GetSiteManager(context);

            Site site = siteManager.GetSite(applicationName);
            if (site != null)
            {
                TestTracer.Trace("{0} Site already exists at {1}. Reusing site", operationName, site.SiteUrl);
                var appManager = new ApplicationManager(siteManager, site, applicationName)
                {
                    SitePoolIndex = siteIndex
                };

                // In site reuse mode, clean out the existing site so we start clean
                // Enumrate all w3wp processes and make sure to kill any process with an open handle to klr.host.dll
                foreach (var process in (await appManager.ProcessManager.GetProcessesAsync()).Where(p => p.Name.Equals("w3wp", StringComparison.OrdinalIgnoreCase)))
                {
                    var extendedProcess = await appManager.ProcessManager.GetProcessAsync(process.Id);
                    if (extendedProcess.OpenFileHandles.Any(h => h.IndexOf("dnx.host.dll", StringComparison.OrdinalIgnoreCase) != -1))
                    {
                        await appManager.ProcessManager.KillProcessAsync(extendedProcess.Id, throwOnError:false);
                    }
                }

                await appManager.RepositoryManager.Delete(deleteWebRoot: true, ignoreErrors: true);

                // Make sure we start with the correct default file as some tests expect it
                WriteIndexHtml(appManager);

                TestTracer.Trace("{0} completed", operationName);
                return appManager;
            }
            else
            {
                TestTracer.Trace("{0} Creating new site", operationName);
                lock (_createSiteLock)
                {
                    if (ConfigurationManager.AppSettings["UseNetworkServiceIdentity"] == "true")
                    {
                        var applicationsPath = context.Configuration.ApplicationsPath;
                        if (!Directory.Exists(applicationsPath))
                        {
                            Directory.CreateDirectory(applicationsPath);

                            var accessRule = new FileSystemAccessRule("NETWORK SERVICE",
                                                 fileSystemRights: FileSystemRights.Modify | FileSystemRights.Write | FileSystemRights.ReadAndExecute | FileSystemRights.Read | FileSystemRights.ListDirectory,
                                                 inheritanceFlags: InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit,
                                                 propagationFlags: PropagationFlags.None,
                                                 type: AccessControlType.Allow);

                            var directoryInfo = new DirectoryInfo(applicationsPath);
                            DirectorySecurity directorySecurity = directoryInfo.GetAccessControl();
                            directorySecurity.AddAccessRule(accessRule);
                            directoryInfo.SetAccessControl(directorySecurity);
                        }
                    }

                    site = siteManager.CreateSiteAsync(applicationName).Result;
                }

                TestTracer.Trace("{0} Created new site at {1}", operationName, site.SiteUrl);
                return new ApplicationManager(siteManager, site, applicationName)
                {
                    SitePoolIndex = siteIndex
                };
            }
        }

        private async Task VerifyTriggeredJobDoesNotTrigger(ApplicationManager appManager, string jobName)
        {
            HttpUnsuccessfulRequestException thrownException = null;
            try
            {
                await appManager.JobsManager.InvokeTriggeredJobAsync(jobName);
            }
            catch (HttpUnsuccessfulRequestException ex)
            {
                thrownException = ex;
            }

            Assert.NotNull(thrownException);
            Assert.Equal(HttpStatusCode.Conflict, thrownException.ResponseMessage.StatusCode);
        }