func TestHeartbeatSignals(t *testing.T) {
	setupTlsConfigs(t)

	for tlsString, tlsConfig := range tlsConfigs {

		testTask, _, err := setupAPITestData(testConfig, evergreen.CompileStage, "linux-64", filepath.Join(testDirectory, "testdata/config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
		testutil.HandleTestingErr(err, t, "Couldn't make test data: %v", err)

		Convey("With a live api server, agent, and test task over "+tlsString, t, func() {
			testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := createAgent(testServer, testTask)
			testutil.HandleTestingErr(err, t, "failed to create agent: %v")
			testAgent.heartbeater.Interval = 100 * time.Millisecond
			testAgent.StartBackgroundActions(&NoopSignalHandler{})

			Convey("killing the server should result in failure signal", func() {
				testServer.Listener.Close()
				signal, ok := <-testAgent.signalHandler.heartbeatChan
				So(ok, ShouldBeTrue)
				So(signal, ShouldEqual, comm.HeartbeatMaxFailed)
			})
		})
	}
}

func TestShellExecuteCommand(t *testing.T) {
	stopper := make(chan bool)
	defer close(stopper)

	testConfig := evergreen.TestConfig()
	server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
	if err != nil {
		t.Fatalf("failed to create test server %+v", err)
	}

	httpCom := plugintest.TestAgentCommunicator("testTaskId", "testTaskSecret", server.URL)
	jsonCom := &comm.TaskJSONCommunicator{"shell", httpCom}

	conf := &model.TaskConfig{Expansions: &command.Expansions{}, Task: &task.Task{}, Project: &model.Project{}}

	Convey("With a shell command", t, func() {

		Convey("if unset, default is determined by local command", func() {
			cmd := &ShellExecCommand{}
			So(cmd.Execute(&plugintest.MockLogger{}, jsonCom, conf, stopper), ShouldBeNil)
			So(cmd.Shell, ShouldEqual, "")
		})

		for _, sh := range []string{"/bin/sh", "/bin/bash", "sh", "bash", "python", "/usr/bin/python"} {
			Convey(fmt.Sprintf("when set, %s is not overwritten during execution", sh), func() {
				cmd := &ShellExecCommand{Shell: sh}
				So(cmd.Execute(&plugintest.MockLogger{}, jsonCom, conf, stopper), ShouldBeNil)
				So(cmd.Shell, ShouldEqual, sh)
			})
		}
	})
}

func TestPatchPluginAPI(t *testing.T) {
	testConfig := evergreen.TestConfig()
	cwd := testutil.GetDirectoryOfFile()
	Convey("With a running api server and installed plugin", t, func() {
		registry := plugin.NewSimpleRegistry()
		gitPlugin := &GitPlugin{}
		err := registry.Register(gitPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register patch plugin")
		server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		taskConfig, _ := plugintest.CreateTestConfig(filepath.Join(cwd, "testdata", "plugin_patch.yml"), t)
		testCommand := GitGetProjectCommand{Directory: "dir"}
		_, _, err = plugintest.SetupAPITestData("testTask", filepath.Join(cwd, "testdata", "testmodule.patch"), t)
		testutil.HandleTestingErr(err, t, "Couldn't set up test documents")
		testTask, err := task.FindOne(task.ById("testTaskId"))
		testutil.HandleTestingErr(err, t, "Couldn't set up test patch task")

		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agentutil.NewTestLogger(sliceAppender)

		Convey("calls to existing tasks with patches should succeed", func() {
			httpCom := plugintest.TestAgentCommunicator(testTask.Id, testTask.Secret, server.URL)
			pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
			patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
			So(err, ShouldBeNil)
			So(patch, ShouldNotBeNil)
			testutil.HandleTestingErr(db.Clear(version.Collection), t,
				"unable to clear versions collection")
		})
		Convey("calls to non-existing tasks should fail", func() {
			v := version.Version{Id: ""}
			testutil.HandleTestingErr(v.Insert(), t, "Couldn't insert dummy version")
			httpCom := plugintest.TestAgentCommunicator("BAD_TASK_ID", "", server.URL)
			pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
			patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
			So(err.Error(), ShouldContainSubstring, "not found")
			So(err, ShouldNotBeNil)
			So(patch, ShouldBeNil)
			testutil.HandleTestingErr(db.Clear(version.Collection), t,
				"unable to clear versions collection")
		})
		Convey("calls to existing tasks without patches should fail", func() {
			noPatchTask := task.Task{Id: "noPatchTask", BuildId: "a"}
			testutil.HandleTestingErr(noPatchTask.Insert(), t, "Couldn't insert patch task")
			noPatchVersion := version.Version{Id: "noPatchVersion", BuildIds: []string{"a"}}
			testutil.HandleTestingErr(noPatchVersion.Insert(), t, "Couldn't insert patch version")
			v := version.Version{Id: ""}
			testutil.HandleTestingErr(v.Insert(), t, "Couldn't insert dummy version")
			httpCom := plugintest.TestAgentCommunicator(noPatchTask.Id, "", server.URL)
			pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
			patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
			So(err, ShouldNotBeNil)
			So(err.Error(), ShouldContainSubstring, "no patch found for task")
			So(patch, ShouldBeNil)
			testutil.HandleTestingErr(db.Clear(version.Collection), t,
				"unable to clear versions collection")
		})

	})
}

func TestFunctionVariantExclusion(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		// test against the windows8 and linux-64 variants; linux-64 excludes a test command
		for _, variant := range []string{"windows8", "linux-64"} {
			Convey("With agent running a "+variant+" task and live API server over "+tlsString, t, func() {
				testTask, _, err := setupAPITestData(testConfig, "variant_test", variant, filepath.Join(testDirectory, "testdata/config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
				testutil.HandleTestingErr(err, t, "Failed to find test task")
				testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
				testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
				testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert, testPidFile)
				So(err, ShouldBeNil)
				So(testAgent, ShouldNotBeNil)
				Convey("running the task", func() {
					testAgent.RunTask()
					testAgent.APILogger.Flush()
					if variant == "windows8" {
						Convey("the variant-specific function command should run", func() {
							So(scanLogsForTask(testTask.Id, "", "variant not excluded!"), ShouldBeTrue)
						})
					} else {
						Convey("the variant-specific function command should not run", func() {
							So(scanLogsForTask(testTask.Id, "", "variant not excluded!"), ShouldBeFalse)
							So(scanLogsForTask(testTask.Id, "", "Skipping command 'shell.exec'"), ShouldBeTrue)
						})
					}
				})
			})
		}
	}
}

func TestProjectTaskExecTimeout(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		Convey("With agent running a slow test and live API server over "+tlsString, t, func() {
			testTask, _, err := setupAPITestData(testConfig, "project_exec_timeout_task", "linux-64", filepath.Join(testDirectory, "testdata/config_test_plugin/project/project-timeout-test.yml"), NoPatch, t)
			testutil.HandleTestingErr(err, t, "Failed to find test task")
			testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert, testPidFile)
			So(err, ShouldBeNil)
			So(testAgent, ShouldNotBeNil)

			Convey("after the slow test runs beyond the project timeout threshold", func() {
				// actually run the task.
				// this function won't return until the whole thing is done.
				testAgent.RunTask()
				testAgent.APILogger.Flush()
				time.Sleep(5 * time.Second)
				printLogsForTask(testTask.Id)
				Convey("the test should be marked as failed and timed out", func() {
					So(scanLogsForTask(testTask.Id, "", "executing the pre-run script"), ShouldBeTrue)
					So(scanLogsForTask(testTask.Id, "", "executing the post-run script!"), ShouldBeTrue)
					So(scanLogsForTask(testTask.Id, "", "executing the task-timeout script!"), ShouldBeTrue)
					testTask, err = task.FindOne(task.ById(testTask.Id))
					So(testTask.Status, ShouldEqual, evergreen.TaskFailed)
					So(testTask.Details.TimedOut, ShouldBeTrue)
					So(testTask.Details.Description, ShouldEqual, "shell.exec")
				})
			})
		})
	}
}

func TestAgentDirectoryFailure(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		Convey("With agent printing directory and live API server over "+tlsString, t, func() {
			testTask, _, err := setupAPITestData(testConfig, "print_dir_task", "linux-64",
				filepath.Join(testDirectory, "testdata", "config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
			testutil.HandleTestingErr(err, t, "Failed to find test task")
			testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert, testPidFile)
			So(err, ShouldBeNil)
			So(testAgent, ShouldNotBeNil)

			dir, err := os.Getwd()

			testutil.HandleTestingErr(err, t, "Failed to read current directory")

			distro, err := testAgent.GetDistro()
			testutil.HandleTestingErr(err, t, "Failed to get agent distro")

			h := md5.New()
			h.Write([]byte(
				fmt.Sprintf("%s_%d_%d", testTask.Id, 0, os.Getpid())))
			dirName := hex.EncodeToString(h.Sum(nil))
			newDir := filepath.Join(distro.WorkDir, dirName)

			newDirFile, err := os.Create(newDir)
			testutil.HandleTestingErr(err, t, "Couldn't create file: %v", err)

			_, err = testAgent.RunTask()
			Convey("Then the agent should have errored", func() {
				So(err, ShouldNotBeNil)
			})

			printLogsForTask(testTask.Id)
			Convey("Then the task should not have been run", func() {
				So(scanLogsForTask(testTask.Id, "", "printing current directory"), ShouldBeFalse)
				So(scanLogsForTask(testTask.Id, "", newDir), ShouldBeFalse)
			})
			<-testAgent.KillChan
			Convey("Then the taskDetail type should have been set to SystemCommandType and have status failed", func() {
				select {
				case detail := <-testAgent.endChan:
					So(detail.Type, ShouldEqual, model.SystemCommandType)
					So(detail.Status, ShouldEqual, evergreen.TaskFailed)
				default:
					t.Errorf("unable to read from the endChan")
				}
			})
			err = os.Chdir(dir)
			testutil.HandleTestingErr(err, t, "Failed to change directory back to main dir")

			testutil.HandleTestingErr(newDirFile.Close(), t, "failed to close dummy directory, file")
			err = os.Remove(newDir)
			testutil.HandleTestingErr(err, t, "Failed to remove dummy directory file")
		})
	}
}

func TestPatchTask(t *testing.T) {
	setupTlsConfigs(t)
	testConfig := evergreen.TestConfig()
	db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))
	patchModes := []patchTestMode{InlinePatch, ExternalPatch}
	testutil.ConfigureIntegrationTest(t, testConfig, "TestPatchTask")
	for tlsString, tlsConfig := range tlsConfigs {
		for _, testSetup := range testSetups {
			Convey(testSetup.testSpec, t, func() {
				Convey("With agent running a patched 'compile'"+tlsString, func() {
					for _, mode := range patchModes {
						Convey(fmt.Sprintf("Using patch mode %v", mode.String()), func() {
							testTask, b, err := setupAPITestData(testConfig, "compile", "linux-64", filepath.Join(testDirectory, "testdata/config_test_plugin/project/evergreen-ci-render.yml"), mode, t)

							githash := "1e5232709595db427893826ce19289461cba3f75"
							setupPatches(mode, b, t,
								patchRequest{"", filepath.Join(testDirectory, "testdata/test.patch"), githash},
								patchRequest{"recursive", filepath.Join(testDirectory, "testdata/testmodule.patch"), githash})

							testutil.HandleTestingErr(err, t, "Error setting up test data: %v", err)
							testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
							testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
							testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert, testPidFile)

							// actually run the task.
							// this function won't return until the whole thing is done.
							testAgent.RunTask()
							time.Sleep(100 * time.Millisecond)
							testAgent.APILogger.FlushAndWait()
							printLogsForTask(testTask.Id)

							Convey("all scripts in task should have been run successfully", func() {
								So(scanLogsForTask(testTask.Id, "", "executing the pre-run script"), ShouldBeTrue)
								So(scanLogsForTask(testTask.Id, "", "executing the post-run script!"), ShouldBeTrue)

								So(scanLogsForTask(testTask.Id, "", "Cloning into") || // git 1.8
									scanLogsForTask(testTask.Id, "", "Initialized empty Git repository"), // git 1.7
									ShouldBeTrue)

								So(scanLogsForTask(testTask.Id, "", "i am patched!"), ShouldBeTrue)
								So(scanLogsForTask(testTask.Id, "", "i am a patched module"), ShouldBeTrue)

								So(scanLogsForTask(testTask.Id, "", "i am compiling!"), ShouldBeTrue)
								So(scanLogsForTask(testTask.Id, "", "i am sanity testing!"), ShouldBeTrue)

								testTask, err = task.FindOne(task.ById(testTask.Id))
								testutil.HandleTestingErr(err, t, "Error finding test task: %v", err)
								So(testTask.Status, ShouldEqual, evergreen.TaskSucceeded)
							})
						})
					}
				})
			})
		}
	}

}

func TestS3CopyPluginExecution(t *testing.T) {

	testConfig := evergreen.TestConfig()
	db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))

	testutil.ConfigureIntegrationTest(t, testConfig, "TestS3CopyPluginExecution")

	Convey("With a SimpleRegistry and test project file", t, func() {
		registry := plugin.NewSimpleRegistry()
		s3CopyPlugin := &S3CopyPlugin{}
		testutil.HandleTestingErr(registry.Register(s3CopyPlugin), t, "failed to register s3Copy plugin")
		testutil.HandleTestingErr(registry.Register(&s3.S3Plugin{}), t, "failed to register S3 plugin")
		testutil.HandleTestingErr(
			db.ClearCollections(model.PushlogCollection, version.Collection), t,
			"error clearing test collections")
		version := &version.Version{
			Id: "",
		}
		So(version.Insert(), ShouldBeNil)
		server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")

		httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)

		//server.InstallPlugin(s3CopyPlugin)

		taskConfig, err := plugintest.CreateTestConfig("testdata/plugin_s3_copy.yml", t)
		testutil.HandleTestingErr(err, t, "failed to create test config: %v", err)
		taskConfig.WorkDir = "."
		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agentutil.NewTestLogger(sliceAppender)

		taskConfig.Expansions.Update(map[string]string{
			"aws_key":    testConfig.Providers.AWS.Id,
			"aws_secret": testConfig.Providers.AWS.Secret,
		})

		Convey("the s3 copy command should execute successfully", func() {
			for _, task := range taskConfig.Project.Tasks {
				So(len(task.Commands), ShouldNotEqual, 0)
				for _, command := range task.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &comm.TaskJSONCommunicator{s3CopyPlugin.Name(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig,
						make(chan bool))
					So(err, ShouldBeNil)
				}
			}
		})
	})
}

func TestPluginFunctions(t *testing.T) {
	testConfig := evergreen.TestConfig()
	testutil.ConfigureIntegrationTest(t, testConfig, "TestPatchTask")
	Convey("With a SimpleRegistry", t, func() {
		Convey("with a project file containing functions", func() {
			registry := plugin.NewSimpleRegistry()
			err := registry.Register(&shell.ShellPlugin{})
			testutil.HandleTestingErr(err, t, "Couldn't register plugin")
			err = registry.Register(&expansions.ExpansionsPlugin{})
			testutil.HandleTestingErr(err, t, "Couldn't register plugin")

			testServer, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
			testutil.HandleTestingErr(err, t, "Couldn't set up testing server")

			taskConfig, err := createTestConfig("testdata/plugin_project_functions.yml", t)
			testutil.HandleTestingErr(err, t, "failed to create test config: %v", err)

			Convey("all commands in project file should parse successfully", func() {
				for _, newTask := range taskConfig.Project.Tasks {
					for _, command := range newTask.Commands {
						pluginCmd, err := registry.GetCommands(command, taskConfig.Project.Functions)
						testutil.HandleTestingErr(err, t, "Got error getting plugin command: %v")
						So(pluginCmd, ShouldNotBeNil)
						So(err, ShouldBeNil)
					}
				}
			})

			httpCom, err := comm.NewHTTPCommunicator(testServer.URL, "mocktaskid", "mocktasksecret", "", nil)
			So(err, ShouldBeNil)
			So(httpCom, ShouldNotBeNil)

			Convey("all commands in test project should execute successfully", func() {
				sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
				logger := agentutil.NewTestLogger(sliceAppender)
				for _, newTask := range taskConfig.Project.Tasks {
					So(len(newTask.Commands), ShouldNotEqual, 0)
					for _, command := range newTask.Commands {
						pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
						testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
						So(pluginCmds, ShouldNotBeNil)
						So(err, ShouldBeNil)
						So(len(pluginCmds), ShouldEqual, 1)
						cmd := pluginCmds[0]
						pluginCom := &comm.TaskJSONCommunicator{cmd.Plugin(), httpCom}
						err = cmd.Execute(logger, pluginCom, taskConfig, make(chan bool))
						So(err, ShouldBeNil)
					}
				}
			})
		})
	})
}

func TestAttachResults(t *testing.T) {
	resetTasks(t)
	testConfig := evergreen.TestConfig()
	cwd := testutil.GetDirectoryOfFile()
	Convey("With attachResults plugin installed into plugin registry", t, func() {
		registry := plugin.NewSimpleRegistry()
		attachPlugin := &AttachPlugin{}
		err := registry.Register(attachPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")

		server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)
		configFile := filepath.Join(cwd, "testdata", "plugin_attach_results.yml")
		resultsLoc := filepath.Join(cwd, "testdata", "plugin_attach_results.json")
		taskConfig, err := plugintest.CreateTestConfig(configFile, t)
		testutil.HandleTestingErr(err, t, "failed to create test config: %v")
		taskConfig.WorkDir = "."
		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agentutil.NewTestLogger(sliceAppender)

		Convey("all commands in test project should execute successfully", func() {
			for _, projTask := range taskConfig.Project.Tasks {
				So(len(projTask.Commands), ShouldNotEqual, 0)
				for _, command := range projTask.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
					So(err, ShouldBeNil)
					testTask, err := task.FindOne(task.ById(httpCom.TaskId))
					testutil.HandleTestingErr(err, t, "Couldn't find task")
					So(testTask, ShouldNotBeNil)
					// ensure test results are exactly as expected
					// attempt to open the file
					reportFile, err := os.Open(resultsLoc)
					testutil.HandleTestingErr(err, t, "Couldn't open report file: '%v'", err)
					results := &task.TestResults{}
					err = util.ReadJSONInto(reportFile, results)
					testutil.HandleTestingErr(err, t, "Couldn't read report file: '%v'", err)
					testResults := *results
					So(testTask.TestResults, ShouldResemble, testResults.Results)
					testutil.HandleTestingErr(err, t, "Couldn't clean up test temp dir")
				}
			}
		})
	})
}

func setupCLITestHarness() cliTestHarness {
	// create a test API server
	testServer, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
	So(err, ShouldBeNil)
	So(
		db.ClearCollections(
			task.Collection,
			build.Collection,
			user.Collection,
			patch.Collection,
			model.ProjectRefCollection,
			artifact.Collection,
		),
		ShouldBeNil)
	So(db.Clear(patch.Collection), ShouldBeNil)
	So(db.Clear(model.ProjectRefCollection), ShouldBeNil)
	So((&user.DBUser{Id: "testuser", APIKey: "testapikey", EmailAddress: "[email protected]"}).Insert(), ShouldBeNil)
	localConfBytes, err := ioutil.ReadFile(filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "sample.yml"))
	So(err, ShouldBeNil)

	projectRef := &model.ProjectRef{
		Identifier:  "sample",
		Owner:       "evergreen-ci",
		Repo:        "sample",
		RepoKind:    "github",
		Branch:      "master",
		RemotePath:  "evergreen.yml",
		LocalConfig: string(localConfBytes),
		Enabled:     true,
		BatchTime:   180,
	}
	So(projectRef.Insert(), ShouldBeNil)

	// create a settings file for the command line client
	settings := model.CLISettings{
		APIServerHost: testServer.URL + "/api",
		UIServerHost:  "http://dev-evg.mongodb.com",
		APIKey:        "testapikey",
		User:          "testuser",
	}
	settingsFile, err := ioutil.TempFile("", "settings")
	So(err, ShouldBeNil)
	settingsBytes, err := yaml.Marshal(settings)
	So(err, ShouldBeNil)
	_, err = settingsFile.Write(settingsBytes)
	So(err, ShouldBeNil)
	settingsFile.Close()
	return cliTestHarness{testServer, settingsFile.Name()}
}

func TestAgentDebugHandler(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		Convey("With an agent that has not been started", t, func() {
			testAgent, err := New("", "task1", "task1", "", testConfig.Api.HttpsCert, testPidFile)
			So(err, ShouldBeNil)
			Convey("no task or command should be listed", func() {
				task, command := taskAndCommand(testAgent)
				So(task, ShouldEqual, "no running task")
				So(command, ShouldEqual, "no running command")
			})
		})
		Convey("With agent running a slow test and live API server over "+tlsString, t, func() {
			testTask, _, err := setupAPITestData(testConfig, "timeout_task", "linux-64",
				filepath.Join(testDirectory, "testdata/config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
			testutil.HandleTestingErr(err, t, "Failed to find test task")
			testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert, testPidFile)
			So(err, ShouldBeNil)
			So(testAgent, ShouldNotBeNil)

			Convey("the agent should return the correct running task, command, and trace", func() {
				// run the slow task and take a debug trace during.
				var stack []byte
				var task, command string
				done := make(chan struct{})
				go func() {
					time.Sleep(time.Second)
					task, command = taskAndCommand(testAgent)
					stack = util.DebugTrace()
					dumpToLogs(task, command, stack, testAgent)
					done <- struct{}{}
				}()
				testAgent.RunTask()
				testAgent.APILogger.Flush()
				<-done
				So(task, ShouldEqual, testTask.Id)
				So(command, ShouldEqual, "shell.exec")
				gcTesting := "testing.RunTests" // we know this will be present in the trace
				So(string(stack), ShouldContainSubstring, gcTesting)
				Convey("which should also be present in the logs", func() {
					So(scanLogsForTask(testTask.Id, "", gcTesting), ShouldBeTrue)
				})
			})
		})
	}
}

func TestTaskAbortion(t *testing.T) {
	setupTlsConfigs(t)

	testutil.ConfigureIntegrationTest(t, testConfig, "TestTaskAbortion")
	for tlsString, tlsConfig := range tlsConfigs {
		for _, testSetup := range testSetups {
			Convey(testSetup.testSpec, t, func() {
				Convey("With agent running a slow test and live API server over "+tlsString, func() {
					testTask, _, err := setupAPITestData(testConfig, "very_slow_task", "linux-64", filepath.Join(testDirectory, "testdata/config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
					testutil.HandleTestingErr(err, t, "Failed to find test task")
					testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
					testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
					testAgent, err := createAgent(testServer, testTask)
					testutil.HandleTestingErr(err, t, "failed to create agent: %v")

					Convey("when the abort signal is triggered on the task", func() {
						go func() {
							// Wait for a few seconds, then switch the task to aborted!
							time.Sleep(3 * time.Second)
							err := model.AbortTask(testTask.Id, "")
							testutil.HandleTestingErr(err, t, "Failed to abort test task")
							fmt.Println("aborted task.")
						}()

						// actually run the task.
						// this function won't return until the whole thing is done.
						_, err := testAgent.RunTask()
						So(err, ShouldBeNil)

						testAgent.APILogger.Flush()
						time.Sleep(1 * time.Second)
						printLogsForTask(testTask.Id)

						Convey("the pre and post-run scripts should have run", func() {
							So(scanLogsForTask(testTask.Id, "", "executing the pre-run script"), ShouldBeTrue)
							So(scanLogsForTask(testTask.Id, "", "executing the post-run script!"), ShouldBeTrue)
							So(scanLogsForTask(testTask.Id, "", "Received abort signal - stopping."), ShouldBeTrue)
							So(scanLogsForTask(testTask.Id, "", "done with very_slow_task!"), ShouldBeFalse)
							testTask, err = task.FindOne(task.ById(testTask.Id))
							testutil.HandleTestingErr(err, t, "Failed to find test task")
							So(testTask.Status, ShouldEqual, evergreen.TaskUndispatched)
						})
					})
				})
			})
		}
	}
}

func TestTaskFailures(t *testing.T) {
	setupTlsConfigs(t)

	testutil.ConfigureIntegrationTest(t, testConfig, "TestTaskFailures")

	for tlsString, tlsConfig := range tlsConfigs {
		for _, testSetup := range testSetups {
			Convey(testSetup.testSpec, t, func() {
				Convey("With agent running a failing test and live API server over "+tlsString, func() {
					testTask, _, err := setupAPITestData(testConfig, "failing_task",
						"linux-64", filepath.Join(testDirectory, "testdata/config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
					testutil.HandleTestingErr(err, t, "Couldn't create test data: %v", err)
					testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
					testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
					testAgent, err := createAgent(testServer, testTask)
					testutil.HandleTestingErr(err, t, "failed to create agent: %v")

					// actually run the task.
					// this function won't return until the whole thing is done.
					testAgent.RunTask()
					time.Sleep(100 * time.Millisecond)
					testAgent.APILogger.FlushAndWait()
					printLogsForTask(testTask.Id)

					Convey("the pre and post-run scripts should have run", func() {
						So(scanLogsForTask(testTask.Id, "", "executing the pre-run script"), ShouldBeTrue)
						So(scanLogsForTask(testTask.Id, "", "executing the post-run script!"), ShouldBeTrue)

						Convey("the task should have run up until its first failure", func() {
							So(scanLogsForTask(testTask.Id, "", "starting failing_task!"), ShouldBeTrue)
							So(scanLogsForTask(testTask.Id, "", "done with failing_task!"), ShouldBeFalse)
						})

						Convey("the tasks's final status should be FAILED", func() {
							testTask, err = task.FindOne(task.ById(testTask.Id))
							testutil.HandleTestingErr(err, t, "Failed to find test task")
							So(testTask.Status, ShouldEqual, evergreen.TaskFailed)
							So(testTask.Details.Status, ShouldEqual, evergreen.TaskFailed)
							So(testTask.Details.Description, ShouldEqual, "failing shell command")
							So(testTask.Details.TimedOut, ShouldBeFalse)
							So(testTask.Details.Type, ShouldEqual, model.SystemCommandType)
						})
					})
				})
			})
		}
	}
}

func TestPatchPlugin(t *testing.T) {
	cwd := testutil.GetDirectoryOfFile()
	testConfig := evergreen.TestConfig()
	db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))
	Convey("With patch plugin installed into plugin registry", t, func() {
		registry := plugin.NewSimpleRegistry()
		gitPlugin := &GitPlugin{}
		err := registry.Register(gitPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin %v")
		testutil.HandleTestingErr(db.Clear(version.Collection), t,
			"unable to clear versions collection")
		version := &version.Version{
			Id: "",
		}
		So(version.Insert(), ShouldBeNil)
		server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("testTaskId", "testTaskSecret", server.URL)

		//sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		sliceAppender := slogger.StdOutAppender()
		logger := agentutil.NewTestLogger(sliceAppender)

		Convey("all commands in test project should execute successfully", func() {
			taskConfig, err := plugintest.CreateTestConfig(filepath.Join(cwd, "testdata", "plugin_patch.yml"), t)
			testutil.HandleTestingErr(err, t, "could not create test config")

			taskConfig.Task.Requester = evergreen.PatchVersionRequester
			_, _, err = plugintest.SetupAPITestData("testTask", filepath.Join(cwd, "testdata", "testmodule.patch"), t)
			testutil.HandleTestingErr(err, t, "Couldn't set up test documents")

			for _, task := range taskConfig.Project.Tasks {
				So(len(task.Commands), ShouldNotEqual, 0)
				for _, command := range task.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
					So(err, ShouldBeNil)
				}
			}
		})
	})
}

func TestGotestPluginWithEnvironmentVariables(t *testing.T) {
	Convey("With gotest plugin installed into plugin registry", t, func() {
		reset(t)
		testConfig := evergreen.TestConfig()
		testutil.ConfigureIntegrationTest(t, testConfig, "TestGotestPluginWithEnvironmentVariables")
		registry := plugin.NewSimpleRegistry()
		testPlugin := &GotestPlugin{}
		err := registry.Register(testPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin %v")

		server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("testTaskId", "testTaskSecret", server.URL)

		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agentutil.NewTestLogger(sliceAppender)

		Convey("test command should get a copy of custom environment variables", func() {
			curWD, err := os.Getwd()
			testutil.HandleTestingErr(err, t, "Couldn't get working directory: %v")
			taskConfig, err := plugintest.CreateTestConfig("testdata/env.yml", t)
			// manually override working directory to the main repo, since this
			// is much easier than copying over the required testing dependencies
			// to a temporary directory
			testutil.HandleTestingErr(err, t, "Couldn't set up test config %v")
			taskConfig.WorkDir = curWD
			_, _, err = plugintest.SetupAPITestData("testTask", "", t)
			testutil.HandleTestingErr(err, t, "Couldn't set up test documents")

			for _, testTask := range taskConfig.Project.Tasks {
				So(len(testTask.Commands), ShouldNotEqual, 0)
				for _, command := range testTask.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))

					So(err, ShouldBeNil)
				}
			}
		})
	})
}

func TestAgentDirectorySuccess(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		Convey("With agent printing directory and live API server over "+tlsString, t, func() {
			testTask, _, err := setupAPITestData(testConfig, "print_dir_task", "linux-64", filepath.Join(testDirectory, "testdata/config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
			testutil.HandleTestingErr(err, t, "Failed to find test task")
			testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert, testPidFile)
			So(err, ShouldBeNil)
			So(testAgent, ShouldNotBeNil)

			dir, err := os.Getwd()

			testutil.HandleTestingErr(err, t, "Failed to read current directory")

			testAgent.RunTask()
			printLogsForTask(testTask.Id)
			distro, err := testAgent.GetDistro()

			testutil.HandleTestingErr(err, t, "Failed to get agent distro")

			h := md5.New()
			h.Write([]byte(
				fmt.Sprintf("%s_%d_%d", testTask.Id, 0, os.Getpid())))
			dirName := hex.EncodeToString(h.Sum(nil))
			newDir := filepath.Join(distro.WorkDir, dirName)

			Convey("Then the directory should have been set and printed", func() {
				So(scanLogsForTask(testTask.Id, "", "printing current directory"), ShouldBeTrue)
				So(scanLogsForTask(testTask.Id, "", newDir), ShouldBeTrue)
			})
			Convey("Then the directory should have been deleted", func() {
				files, err := ioutil.ReadDir("./")
				testutil.HandleTestingErr(err, t, "Failed to read current directory")
				for _, f := range files {
					So(f.Name(), ShouldNotEqual, newDir)
				}
			})
			err = os.Chdir(dir)
			testutil.HandleTestingErr(err, t, "Failed to change directory back to main dir")
		})
	}
}

func TestPluginExecution(t *testing.T) {
	Convey("With a SimpleRegistry and test project file", t, func() {
		registry := plugin.NewSimpleRegistry()

		plugins := []plugin.CommandPlugin{&MockPlugin{}, &expansions.ExpansionsPlugin{}, &shell.ShellPlugin{}}
		apiPlugins := []plugin.APIPlugin{&MockPlugin{}, &expansions.ExpansionsPlugin{}}
		for _, p := range plugins {
			err := registry.Register(p)
			testutil.HandleTestingErr(err, t, "failed to register plugin")
		}

		testServer, err := service.CreateTestServer(evergreen.TestConfig(), nil, apiPlugins, false)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")

		httpCom, err := comm.NewHTTPCommunicator(testServer.URL, "mocktaskid", "mocktasksecret", "", nil)
		So(err, ShouldBeNil)
		So(httpCom, ShouldNotBeNil)

		pluginConfigPath := filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "plugin_project.yml")