func TestFunctionVariantExclusion(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		// test against the windows8 and linux-64 variants; linux-64 excludes a test command
		for _, variant := range []string{"windows8", "linux-64"} {
			Convey("With agent running a "+variant+" task and live API server over "+tlsString, t, func() {
				testTask, _, err := setupAPITestData(testConfig, "variant_test", variant, filepath.Join(testDirectory, "testdata/config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
				testutil.HandleTestingErr(err, t, "Failed to find test task")
				testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
				testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
				testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert, testPidFile)
				So(err, ShouldBeNil)
				So(testAgent, ShouldNotBeNil)
				Convey("running the task", func() {
					testAgent.RunTask()
					testAgent.APILogger.Flush()
					if variant == "windows8" {
						Convey("the variant-specific function command should run", func() {
							So(scanLogsForTask(testTask.Id, "", "variant not excluded!"), ShouldBeTrue)
						})
					} else {
						Convey("the variant-specific function command should not run", func() {
							So(scanLogsForTask(testTask.Id, "", "variant not excluded!"), ShouldBeFalse)
							So(scanLogsForTask(testTask.Id, "", "Skipping command 'shell.exec'"), ShouldBeTrue)
						})
					}
				})
			})
		}
	}
}

func TestTaskTimeout(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		Convey("With agent running a slow test and live API server over "+tlsString, t, func() {
			testTask, _, err := setupAPITestData(testConfig, "timeout_task", "linux-64", filepath.Join(testDirectory, "testdata/config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
			testutil.HandleTestingErr(err, t, "Failed to find test task")
			testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert, testPidFile)
			So(err, ShouldBeNil)
			So(testAgent, ShouldNotBeNil)

			Convey("after the slow test runs beyond the timeout threshold", func() {
				// actually run the task.
				// this function won't return until the whole thing is done.
				testAgent.RunTask()
				testAgent.APILogger.Flush()
				time.Sleep(5 * time.Second)
				//printLogsForTask(testTask.Id)
				Convey("the test should be marked as failed and timed out", func() {
					So(scanLogsForTask(testTask.Id, "", "executing the pre-run script"), ShouldBeTrue)
					So(scanLogsForTask(testTask.Id, "", "executing the post-run script!"), ShouldBeTrue)
					So(scanLogsForTask(testTask.Id, "", "executing the task-timeout script!"), ShouldBeTrue)
					testTask, err = task.FindOne(task.ById(testTask.Id))
					So(testTask.Status, ShouldEqual, evergreen.TaskFailed)
					So(testTask.Details.TimedOut, ShouldBeTrue)
					So(testTask.Details.Description, ShouldEqual, "shell.exec")
				})
			})
		})
	}
}

func TestParserOnRealTests(t *testing.T) {
	// there are some issues with gccgo:
	testutil.SkipTestUnlessAll(t, "TestParserOnRealTests")

	var parser Parser

	Convey("With a parser", t, func() {
		parser = &VanillaParser{}
		Convey("and some real test output", func() {
			cmd := exec.Command("go", "test", "-v", "./.")
			stdout, err := cmd.StdoutPipe()
			testutil.HandleTestingErr(err, t, "error getting stdout pipe %v")
			testutil.HandleTestingErr(cmd.Start(), t, "couldn't run tests %v")
			err = parser.Parse(stdout)
			testutil.HandleTestingErr(cmd.Wait(), t, "error waiting on test %v")

			Convey("the parser should run successfully", func() {
				So(err, ShouldBeNil)

				Convey("and all results should line up with the logs", func() {
					for _, result := range parser.Results() {
						matchResultWithLog(result, parser.Logs())
					}
				})
			})
		})
	})
}

func TestHeartbeatSignals(t *testing.T) {
	setupTlsConfigs(t)

	for tlsString, tlsConfig := range tlsConfigs {

		testTask, _, err := setupAPITestData(testConfig, evergreen.CompileStage, "linux-64", NoPatch, t)
		testutil.HandleTestingErr(err, t, "Couldn't make test data: %v", err)

		Convey("With a live api server, agent, and test task over "+tlsString, t, func() {
			testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := createAgent(testServer, testTask)
			testutil.HandleTestingErr(err, t, "failed to create agent: %v")
			testAgent.heartbeater.Interval = 100 * time.Millisecond
			testAgent.StartBackgroundActions(&NoopSignalHandler{})

			Convey("killing the server should result in failure signal", func() {
				testServer.Listener.Close()
				signal, ok := <-testAgent.signalHandler.heartbeatChan
				So(ok, ShouldBeTrue)
				So(signal, ShouldEqual, HeartbeatMaxFailed)
			})
		})
	}
}

func TestFindRunningSpawnedHosts(t *testing.T) {
	testConfig := evergreen.TestConfig()
	db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))

	testutil.HandleTestingErr(db.Clear(Collection), t, "Error"+
		" clearing '%v' collection", Collection)

	Convey("With calling FindRunningSpawnedHosts...", t, func() {
		Convey("if there are no spawned hosts, nothing should be returned",
			func() {
				spawnedHosts, err := Find(IsRunningAndSpawned)
				So(err, ShouldBeNil)
				// make sure we only returned no document
				So(len(spawnedHosts), ShouldEqual, 0)

			})

		Convey("if there are spawned hosts, they should be returned", func() {
			host := &Host{}
			host.Id = "spawned-1"
			host.Status = "running"
			host.StartedBy = "user1"
			testutil.HandleTestingErr(host.Insert(), t, "error from "+
				"FindRunningSpawnedHosts")
			spawnedHosts, err := Find(IsRunningAndSpawned)
			testutil.HandleTestingErr(err, t, "error from "+
				"FindRunningSpawnedHosts: %v", err)
			// make sure we only returned no document
			So(len(spawnedHosts), ShouldEqual, 1)

		})
	})
}

func createTestConfig(filename string, t *testing.T) (*model.TaskConfig, error) {
	clearDataMsg := "Failed to clear test data collection"
	testutil.HandleTestingErr(
		db.ClearCollections(
			task.Collection, model.ProjectVarsCollection),
		t, clearDataMsg)

	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	testProject := &model.Project{}
	err = yaml.Unmarshal(data, testProject)
	if err != nil {
		return nil, err
	}

	testProjectRef := &model.ProjectRef{
		Identifier: "mongodb-mongo-master",
		Owner:      "mongodb",
		Repo:       "mongo",
		RepoKind:   "github",
		Branch:     "master",
		Enabled:    true,
		BatchTime:  180,
	}

	workDir, err := ioutil.TempDir("", "plugintest_")
	if err != nil {
		return nil, err
	}

	testTask := &task.Task{
		Id:           "mocktaskid",
		BuildId:      "testBuildId",
		BuildVariant: "linux-64",
		Project:      "mongodb-mongo-master",
		DisplayName:  "test",
		HostId:       "testHost",
		Version:      "versionId",
		Secret:       "mocktasksecret",
		Status:       evergreen.TaskDispatched,
		Revision:     "cb91350bf017337a734dcd0321bf4e6c34990b6a",
		Requester:    evergreen.RepotrackerVersionRequester,
	}
	testutil.HandleTestingErr(testTask.Insert(), t, "failed to insert task")

	projectVars := &model.ProjectVars{
		Id: "mongodb-mongo-master",
		Vars: map[string]string{
			"abc": "xyz",
			"123": "456",
		},
	}
	_, err = projectVars.Upsert()
	testutil.HandleTestingErr(err, t, "failed to upsert project vars")
	testDistro := &distro.Distro{Id: "linux-64", WorkDir: workDir}
	testVersion := &version.Version{}
	return model.NewTaskConfig(testDistro, testVersion, testProject, testTask, testProjectRef)
}

func TestTaskExecTimeout(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		Convey("With agent running a slow test and live API server over "+tlsString, t, func() {
			testTask, _, err := setupAPITestData(testConfig, "exec_timeout_task", "linux-64", NoPatch, t)
			testutil.HandleTestingErr(err, t, "Failed to find test task")
			testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Expansions["api_httpscert"])
			So(err, ShouldBeNil)
			So(testAgent, ShouldNotBeNil)

			Convey("after the slow test runs beyond the timeout threshold", func() {
				// actually run the task.
				// this function won't return until the whole thing is done.
				testAgent.RunTask()
				testAgent.APILogger.Flush()
				time.Sleep(5 * time.Second)
				printLogsForTask(testTask.Id)
				Convey("the test should be marked as failed and timed out", func() {
					So(scanLogsForTask(testTask.Id, "executing the pre-run script"), ShouldBeTrue)
					So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)
					So(scanLogsForTask(testTask.Id, "executing the task-timeout script!"), ShouldBeTrue)
					testTask, err = model.FindTask(testTask.Id)
					So(testTask.Status, ShouldEqual, evergreen.TaskFailed)
					So(testTask.Details.TimedOut, ShouldBeTrue)
					So(testTask.Details.Description, ShouldEqual, "shell.exec")
				})
			})
		})
	}
}

func TestRegistry(t *testing.T) {
	Convey("With a SimpleRegistry", t, func() {
		Convey("Registering a plugin twice should return err", func() {
			registry := plugin.NewSimpleRegistry()
			err := registry.Register(&MockPlugin{})
			testutil.HandleTestingErr(err, t, "Couldn't register plugin")
			err = registry.Register(&shell.ShellPlugin{})
			testutil.HandleTestingErr(err, t, "Couldn't register plugin")
			err = registry.Register(&expansions.ExpansionsPlugin{})
			testutil.HandleTestingErr(err, t, "Couldn't register plugin")
		})
		Convey("with a project file containing references to a valid plugin", func() {
			registry := plugin.NewSimpleRegistry()
			registry.Register(&MockPlugin{})
			registry.Register(&shell.ShellPlugin{})
			registry.Register(&expansions.ExpansionsPlugin{})
			data, err := ioutil.ReadFile("testdata/plugin_project.yml")
			testutil.HandleTestingErr(err, t, "failed to load test yaml file")
			project := &model.Project{}
			err = yaml.Unmarshal(data, project)
			Convey("all commands in project file should load parse successfully", func() {
				for _, newTask := range project.Tasks {
					for _, command := range newTask.Commands {
						pluginCmds, err := registry.GetCommands(command, project.Functions)
						testutil.HandleTestingErr(err, t, "Got error getting plugin commands: %v")
						So(pluginCmds, ShouldNotBeNil)
						So(err, ShouldBeNil)
					}
				}
			})
		})
	})
}

func setupPatches(patchMode patchTestMode, b *build.Build, t *testing.T, patches ...patchRequest) {
	if patchMode == NoPatch {
		return
	}

	ptch := &patch.Patch{
		Status:  evergreen.PatchCreated,
		Version: b.Version,
		Patches: []patch.ModulePatch{},
	}

	for _, p := range patches {
		patchContent, err := ioutil.ReadFile(p.filePath)
		testutil.HandleTestingErr(err, t, "failed to read test patch file")

		if patchMode == InlinePatch {
			ptch.Patches = append(ptch.Patches, patch.ModulePatch{
				ModuleName: p.moduleName,
				Githash:    p.githash,
				PatchSet:   patch.PatchSet{Patch: string(patchContent)},
			})
		} else {
			pId := bson.NewObjectId().Hex()
			So(dbutil.WriteGridFile(patch.GridFSPrefix, pId, strings.NewReader(string(patchContent))), ShouldBeNil)
			ptch.Patches = append(ptch.Patches, patch.ModulePatch{
				ModuleName: p.moduleName,
				Githash:    p.githash,
				PatchSet:   patch.PatchSet{PatchFileId: pId},
			})
		}
	}
	testutil.HandleTestingErr(ptch.Insert(), t, "failed to insert patch")
}

func dropTestDB(t *testing.T) {
	session, _, err := db.GetGlobalSessionFactory().GetSession()
	testutil.HandleTestingErr(err, t, "Error opening database session")
	defer session.Close()
	testutil.HandleTestingErr(session.DB(testConfig.Db).DropDatabase(), t, "Error "+
		"dropping test database")
}

func TestGitPlugin(t *testing.T) {
	Convey("With git plugin installed into plugin registry", t, func() {
		registry := plugin.NewSimpleRegistry()
		gitPlugin := &GitPlugin{}
		err := registry.Register(gitPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")

		server, err := apiserver.CreateTestServer(evergreen.TestConfig(), nil, plugin.Published, false)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)

		taskConfig, err := plugintest.CreateTestConfig("testdata/plugin_clone.yml", t)
		testutil.HandleTestingErr(err, t, "failed to create test config")
		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agent.NewTestLogger(sliceAppender)

		Convey("all commands in test project should execute successfully", func() {
			for _, task := range taskConfig.Project.Tasks {
				So(len(task.Commands), ShouldNotEqual, 0)
				for _, command := range task.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &agent.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
					So(err, ShouldBeNil)
				}
			}
			err = os.RemoveAll(taskConfig.WorkDir)
			testutil.HandleTestingErr(err, t, "Couldn't clean up test temp dir")
		})
	})
}

func TestArchiveExtract(t *testing.T) {
	Convey("After extracting a tarball", t, func() {
		testDir := testutil.GetDirectoryOfFile()
		//Remove the test output dir, in case it was left over from prior test
		err := os.RemoveAll(filepath.Join(testDir, "testdata", "artifacts_test"))
		testutil.HandleTestingErr(err, t, "Couldn't remove test dir")

		f, gz, tarReader, err := TarGzReader(filepath.Join(testDir, "testdata", "artifacts.tar.gz"))
		testutil.HandleTestingErr(err, t, "Couldn't open test tarball")
		defer f.Close()
		defer gz.Close()

		err = Extract(tarReader, filepath.Join(testDir, "testdata", "artifacts_test"))
		So(err, ShouldBeNil)

		Convey("extracted data should match the archive contents", func() {
			f, err := os.Open(filepath.Join(testDir, "testdata", "artifacts_test", "artifacts", "dir1", "dir2", "testfile.txt"))
			So(err, ShouldBeNil)
			defer f.Close()
			data, err := ioutil.ReadAll(f)
			So(err, ShouldBeNil)
			So(string(data), ShouldEqual, "test\n")
		})
	})
}

func TestAgentDebugHandler(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		Convey("With an agent that has not been started", t, func() {
			testAgent, err := New("", "task1", "task1", "", testConfig.Api.HttpsCert)
			So(err, ShouldBeNil)
			Convey("no task or command should be listed", func() {
				task, command := taskAndCommand(testAgent)
				So(task, ShouldEqual, "no running task")
				So(command, ShouldEqual, "no running command")
			})
		})
		Convey("With agent running a slow test and live API server over "+tlsString, t, func() {
			testTask, _, err := setupAPITestData(testConfig, "timeout_task", "linux-64", "testdata/config_test_plugin/project/evergreen-ci-render.yml", NoPatch, t)
			testutil.HandleTestingErr(err, t, "Failed to find test task")
			testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert)
			So(err, ShouldBeNil)
			So(testAgent, ShouldNotBeNil)

			Convey("the agent should return the correct running task, command, and trace", func() {
				// run the slow task and take a debug trace during.
				var stack []byte
				var task, command string
				go func() {
					time.Sleep(time.Second)
					task, command = taskAndCommand(testAgent)
					stack = trace()
					dumpToLogs(task, command, stack, testAgent)
				}()
				testAgent.RunTask()
				testAgent.APILogger.Flush()
				time.Sleep(5 * time.Second)
				So(task, ShouldEqual, testTask.Id)
				So(command, ShouldEqual, "shell.exec")
				// we need to check for two kinds of stacktrace forms, to support GC and GCCGO traces
				gcExecute := "(*ShellExecCommand).Execute"
				gccExecute := "evergreen_plugin_builtin_shell.Execute"
				gcAgent := "(*Agent).RunTask"
				gccAgent := "agent.Agent"
				executeIdx := strings.Index(string(stack), gcExecute) + strings.Index(string(stack), gccExecute)
				So(executeIdx, ShouldBeGreaterThan, 0)
				agentIdx := strings.Index(string(stack), gcAgent) + strings.Index(string(stack), gccAgent)
				So(agentIdx, ShouldBeGreaterThan, 0)
				Convey("which should also be present in the logs", func() {
					So(
						scanLogsForTask(testTask.Id, gcExecute) || scanLogsForTask(testTask.Id, gccExecute),
						ShouldBeTrue,
					)
					So(
						scanLogsForTask(testTask.Id, gcAgent) || scanLogsForTask(testTask.Id, gccAgent),
						ShouldBeTrue,
					)
				})
			})
		})
	}
}

func TestPatchTask(t *testing.T) {
	setupTlsConfigs(t)
	testConfig := evergreen.TestConfig()
	db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))
	patchModes := []patchTestMode{InlinePatch, ExternalPatch}
	testutil.ConfigureIntegrationTest(t, testConfig, "TestPatchTask")
	for tlsString, tlsConfig := range tlsConfigs {
		for _, testSetup := range testSetups {
			Convey(testSetup.testSpec, t, func() {
				Convey("With agent running a patched 'compile'"+tlsString, func() {
					for _, mode := range patchModes {
						Convey(fmt.Sprintf("Using patch mode %v", mode.String()), func() {
							testTask, b, err := setupAPITestData(testConfig, "compile", "linux-64", "testdata/config_test_plugin/project/evergreen-ci-render.yml", mode, t)

							githash := "1e5232709595db427893826ce19289461cba3f75"
							setupPatches(mode, b, t,
								patchRequest{"", "testdata/test.patch", githash},
								patchRequest{"recursive", "testdata/testmodule.patch", githash})

							testutil.HandleTestingErr(err, t, "Error setting up test data: %v", err)
							testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
							testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
							testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert)

							// actually run the task.
							// this function won't return until the whole thing is done.
							testAgent.RunTask()
							time.Sleep(100 * time.Millisecond)
							testAgent.APILogger.FlushAndWait()
							printLogsForTask(testTask.Id)

							Convey("all scripts in task should have been run successfully", func() {
								So(scanLogsForTask(testTask.Id, "executing the pre-run script"), ShouldBeTrue)
								So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)

								So(scanLogsForTask(testTask.Id, "Cloning into") || // git 1.8
									scanLogsForTask(testTask.Id, "Initialized empty Git repository"), // git 1.7
									ShouldBeTrue)

								So(scanLogsForTask(testTask.Id, "i am patched!"), ShouldBeTrue)
								So(scanLogsForTask(testTask.Id, "i am a patched module"), ShouldBeTrue)

								So(scanLogsForTask(testTask.Id, "i am compiling!"), ShouldBeTrue)
								So(scanLogsForTask(testTask.Id, "i am sanity testing!"), ShouldBeTrue)

								testTask, err = task.FindOne(task.ById(testTask.Id))
								testutil.HandleTestingErr(err, t, "Error finding test task: %v", err)
								So(testTask.Status, ShouldEqual, evergreen.TaskSucceeded)
							})
						})
					}
				})
			})
		}
	}

}

func TestDeletingBuild(t *testing.T) {

	Convey("With a build", t, func() {

		testutil.HandleTestingErr(db.Clear(build.Collection), t, "Error clearing"+
			" '%v' collection", build.Collection)

		b := &build.Build{
			Id: "build",
		}
		So(b.Insert(), ShouldBeNil)

		Convey("deleting it should remove it and all its associated"+
			" tasks from the database", func() {

			testutil.HandleTestingErr(db.ClearCollections(task.Collection), t, "Error"+
				" clearing '%v' collection", task.Collection)

			// insert two tasks that are part of the build, and one that isn't
			matchingTaskOne := &task.Task{
				Id:      "matchingOne",
				BuildId: b.Id,
			}
			So(matchingTaskOne.Insert(), ShouldBeNil)

			matchingTaskTwo := &task.Task{
				Id:      "matchingTwo",
				BuildId: b.Id,
			}
			So(matchingTaskTwo.Insert(), ShouldBeNil)

			nonMatchingTask := &task.Task{
				Id:      "nonMatching",
				BuildId: "blech",
			}
			So(nonMatchingTask.Insert(), ShouldBeNil)

			// delete the build, make sure only it and its tasks are deleted

			So(DeleteBuild(b.Id), ShouldBeNil)

			b, err := build.FindOne(build.ById(b.Id))
			So(err, ShouldBeNil)
			So(b, ShouldBeNil)

			matchingTasks, err := task.Find(task.ByBuildId("build"))
			So(err, ShouldBeNil)
			So(len(matchingTasks), ShouldEqual, 0)

			nonMatchingTask, err = task.FindOne(task.ById(nonMatchingTask.Id))
			So(err, ShouldBeNil)
			So(nonMatchingTask, ShouldNotBeNil)
		})
	})
}

// prependConfigToVersion modifies the project config with the given id
func prependConfigToVersion(t *testing.T, versionId, configData string) {
	v, err := version.FindOne(version.ById(versionId))
	testutil.HandleTestingErr(err, t, "failed to load version")
	if v == nil {
		err = fmt.Errorf("could not find version to update")
		testutil.HandleTestingErr(err, t, "failed to find version")
	}
	v.Config = configData + v.Config
	testutil.HandleTestingErr(dbutil.ClearCollections(version.Collection), t, "couldnt reset version")
	testutil.HandleTestingErr(v.Insert(), t, "failed to insert version")
}

func TestWriteToTempFile(t *testing.T) {
	Convey("When writing content to a temp file", t, func() {
		Convey("ensure the exact contents passed are written", func() {
			fileData := "data"
			filePath, err := WriteToTempFile(fileData)
			testutil.HandleTestingErr(err, t, "error writing to temp file %v")
			fileBytes, err := ioutil.ReadFile(filePath)
			testutil.HandleTestingErr(err, t, "error reading from temp file %v")
			So(string(fileBytes), ShouldEqual, fileData)
			testutil.HandleTestingErr(os.Remove(filePath), t,
				"error removing to temp file %v")
		})
	})
}

func TestAgentDirectoryFailure(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		Convey("With agent printing directory and live API server over "+tlsString, t, func() {
			testTask, _, err := setupAPITestData(testConfig, "print_dir_task", "linux-64",
				filepath.Join(testDirectory, "testdata", "config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
			testutil.HandleTestingErr(err, t, "Failed to find test task")
			testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert, testPidFile)
			So(err, ShouldBeNil)
			So(testAgent, ShouldNotBeNil)

			dir, err := os.Getwd()

			testutil.HandleTestingErr(err, t, "Failed to read current directory")

			distro, err := testAgent.GetDistro()
			testutil.HandleTestingErr(err, t, "Failed to get agent distro")

			h := md5.New()
			h.Write([]byte(
				fmt.Sprintf("%s_%d_%d", testTask.Id, 0, os.Getpid())))
			dirName := hex.EncodeToString(h.Sum(nil))
			newDir := filepath.Join(distro.WorkDir, dirName)

			newDirFile, err := os.Create(newDir)
			testutil.HandleTestingErr(err, t, "Couldn't create file: %v", err)

			_, err = testAgent.RunTask()
			Convey("Then the agent should have errored", func() {
				So(err, ShouldNotBeNil)
			})

			printLogsForTask(testTask.Id)
			Convey("Then the task should not have been run", func() {
				So(scanLogsForTask(testTask.Id, "", "printing current directory"), ShouldBeFalse)
				So(scanLogsForTask(testTask.Id, "", newDir), ShouldBeFalse)
			})
			<-testAgent.KillChan
			Convey("Then the taskDetail type should have been set to SystemCommandType and have status failed", func() {
				select {
				case detail := <-testAgent.endChan:
					So(detail.Type, ShouldEqual, model.SystemCommandType)
					So(detail.Status, ShouldEqual, evergreen.TaskFailed)
				default:
					t.Errorf("unable to read from the endChan")
				}
			})
			err = os.Chdir(dir)
			testutil.HandleTestingErr(err, t, "Failed to change directory back to main dir")

			testutil.HandleTestingErr(newDirFile.Close(), t, "failed to close dummy directory, file")
			err = os.Remove(newDir)
			testutil.HandleTestingErr(err, t, "Failed to remove dummy directory file")
		})
	}
}

func TestAgentDebugHandler(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		Convey("With an agent that has not been started", t, func() {
			testAgent, err := New("", "task1", "task1", "", testConfig.Api.HttpsCert, testPidFile)
			So(err, ShouldBeNil)
			Convey("no task or command should be listed", func() {
				task, command := taskAndCommand(testAgent)
				So(task, ShouldEqual, "no running task")
				So(command, ShouldEqual, "no running command")
			})
		})
		Convey("With agent running a slow test and live API server over "+tlsString, t, func() {
			testTask, _, err := setupAPITestData(testConfig, "timeout_task", "linux-64",
				filepath.Join(testDirectory, "testdata/config_test_plugin/project/evergreen-ci-render.yml"), NoPatch, t)
			testutil.HandleTestingErr(err, t, "Failed to find test task")
			testServer, err := service.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert, testPidFile)
			So(err, ShouldBeNil)
			So(testAgent, ShouldNotBeNil)

			Convey("the agent should return the correct running task, command, and trace", func() {
				// run the slow task and take a debug trace during.
				var stack []byte
				var task, command string
				done := make(chan struct{})
				go func() {
					time.Sleep(time.Second)
					task, command = taskAndCommand(testAgent)
					stack = util.DebugTrace()
					dumpToLogs(task, command, stack, testAgent)
					done <- struct{}{}
				}()
				testAgent.RunTask()
				testAgent.APILogger.Flush()
				<-done
				So(task, ShouldEqual, testTask.Id)
				So(command, ShouldEqual, "shell.exec")
				gcTesting := "testing.RunTests" // we know this will be present in the trace
				So(string(stack), ShouldContainSubstring, gcTesting)
				Convey("which should also be present in the logs", func() {
					So(scanLogsForTask(testTask.Id, "", gcTesting), ShouldBeTrue)
				})
			})
		})
	}
}

func TestTaskAbortion(t *testing.T) {
	setupTlsConfigs(t)

	testutil.ConfigureIntegrationTest(t, testConfig, "TestTaskAbortion")
	for tlsString, tlsConfig := range tlsConfigs {
		for _, testSetup := range testSetups {
			Convey(testSetup.testSpec, t, func() {
				Convey("With agent running a slow test and live API server over "+tlsString, func() {
					testTask, _, err := setupAPITestData(testConfig, "very_slow_task", "linux-64", NoPatch, t)
					testutil.HandleTestingErr(err, t, "Failed to find test task")
					testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
					testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
					testAgent, err := createAgent(testServer, testTask)
					testutil.HandleTestingErr(err, t, "failed to create agent: %v")

					Convey("when the abort signal is triggered on the task", func() {
						go func() {
							// Wait for a few seconds, then switch the task to aborted!
							time.Sleep(3 * time.Second)
							err := testTask.Abort("", true)
							testutil.HandleTestingErr(err, t, "Failed to abort test task")
							fmt.Println("aborted task.")
						}()

						// actually run the task.
						// this function won't return until the whole thing is done.
						_, err := testAgent.RunTask()
						So(err, ShouldBeNil)

						testAgent.APILogger.Flush()
						time.Sleep(1 * time.Second)
						printLogsForTask(testTask.Id)

						Convey("the pre and post-run scripts should have run", func() {
							So(scanLogsForTask(testTask.Id, "executing the pre-run script"), ShouldBeTrue)
							So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)
							So(scanLogsForTask(testTask.Id, "Received abort signal - stopping."), ShouldBeTrue)
							So(scanLogsForTask(testTask.Id, "done with very_slow_task!"), ShouldBeFalse)
							testTask, err = model.FindTask(testTask.Id)
							testutil.HandleTestingErr(err, t, "Failed to find test task")
							So(testTask.Status, ShouldEqual, evergreen.TaskUndispatched)
						})
					})
				})
			})
		}
	}
}