func TestRegistry(t *testing.T) {
	Convey("With a SimpleRegistry", t, func() {
		Convey("Registering a plugin twice should return err", func() {
			registry := plugin.NewSimpleRegistry()
			err := registry.Register(&MockPlugin{})
			testutil.HandleTestingErr(err, t, "Couldn't register plugin")
			err = registry.Register(&shell.ShellPlugin{})
			testutil.HandleTestingErr(err, t, "Couldn't register plugin")
			err = registry.Register(&expansions.ExpansionsPlugin{})
			testutil.HandleTestingErr(err, t, "Couldn't register plugin")
		})
		Convey("with a project file containing references to a valid plugin", func() {
			registry := plugin.NewSimpleRegistry()
			registry.Register(&MockPlugin{})
			registry.Register(&shell.ShellPlugin{})
			registry.Register(&expansions.ExpansionsPlugin{})
			data, err := ioutil.ReadFile("testdata/plugin_project.yml")
			testutil.HandleTestingErr(err, t, "failed to load test yaml file")
			project := &model.Project{}
			err = yaml.Unmarshal(data, project)
			Convey("all commands in project file should load parse successfully", func() {
				for _, newTask := range project.Tasks {
					for _, command := range newTask.Commands {
						pluginCmds, err := registry.GetCommands(command, project.Functions)
						testutil.HandleTestingErr(err, t, "Got error getting plugin commands: %v")
						So(pluginCmds, ShouldNotBeNil)
						So(err, ShouldBeNil)
					}
				}
			})
		})
	})
}

func TestGitPlugin(t *testing.T) {
	Convey("With git plugin installed into plugin registry", t, func() {
		registry := plugin.NewSimpleRegistry()
		gitPlugin := &GitPlugin{}
		err := registry.Register(gitPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")

		server, err := apiserver.CreateTestServer(evergreen.TestConfig(), nil, plugin.Published, false)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)

		taskConfig, err := plugintest.CreateTestConfig("testdata/plugin_clone.yml", t)
		testutil.HandleTestingErr(err, t, "failed to create test config")
		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agent.NewTestLogger(sliceAppender)

		Convey("all commands in test project should execute successfully", func() {
			for _, task := range taskConfig.Project.Tasks {
				So(len(task.Commands), ShouldNotEqual, 0)
				for _, command := range task.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &agent.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
					So(err, ShouldBeNil)
				}
			}
			err = os.RemoveAll(taskConfig.WorkDir)
			testutil.HandleTestingErr(err, t, "Couldn't clean up test temp dir")
		})
	})
}

func TestPatchPluginAPI(t *testing.T) {
	testConfig := evergreen.TestConfig()
	cwd := testutil.GetDirectoryOfFile()
	Convey("With a running api server and installed plugin", t, func() {
		registry := plugin.NewSimpleRegistry()
		gitPlugin := &GitPlugin{}
		err := registry.Register(gitPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register patch plugin")
		server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		taskConfig, _ := plugintest.CreateTestConfig(filepath.Join(cwd, "testdata", "plugin_patch.yml"), t)
		testCommand := GitGetProjectCommand{Directory: "dir"}
		_, _, err = plugintest.SetupAPITestData("testTask", filepath.Join(cwd, "testdata", "testmodule.patch"), t)
		testutil.HandleTestingErr(err, t, "Couldn't set up test documents")
		testTask, err := task.FindOne(task.ById("testTaskId"))
		testutil.HandleTestingErr(err, t, "Couldn't set up test patch task")

		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agentutil.NewTestLogger(sliceAppender)

		Convey("calls to existing tasks with patches should succeed", func() {
			httpCom := plugintest.TestAgentCommunicator(testTask.Id, testTask.Secret, server.URL)
			pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
			patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
			So(err, ShouldBeNil)
			So(patch, ShouldNotBeNil)
			testutil.HandleTestingErr(db.Clear(version.Collection), t,
				"unable to clear versions collection")
		})
		Convey("calls to non-existing tasks should fail", func() {
			v := version.Version{Id: ""}
			testutil.HandleTestingErr(v.Insert(), t, "Couldn't insert dummy version")
			httpCom := plugintest.TestAgentCommunicator("BAD_TASK_ID", "", server.URL)
			pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
			patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
			So(err.Error(), ShouldContainSubstring, "not found")
			So(err, ShouldNotBeNil)
			So(patch, ShouldBeNil)
			testutil.HandleTestingErr(db.Clear(version.Collection), t,
				"unable to clear versions collection")
		})
		Convey("calls to existing tasks without patches should fail", func() {
			noPatchTask := task.Task{Id: "noPatchTask", BuildId: "a"}
			testutil.HandleTestingErr(noPatchTask.Insert(), t, "Couldn't insert patch task")
			noPatchVersion := version.Version{Id: "noPatchVersion", BuildIds: []string{"a"}}
			testutil.HandleTestingErr(noPatchVersion.Insert(), t, "Couldn't insert patch version")
			v := version.Version{Id: ""}
			testutil.HandleTestingErr(v.Insert(), t, "Couldn't insert dummy version")
			httpCom := plugintest.TestAgentCommunicator(noPatchTask.Id, "", server.URL)
			pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
			patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
			So(err, ShouldNotBeNil)
			So(err.Error(), ShouldContainSubstring, "no patch found for task")
			So(patch, ShouldBeNil)
			testutil.HandleTestingErr(db.Clear(version.Collection), t,
				"unable to clear versions collection")
		})

	})
}

// New creates a new agent to run a given task.
func New(apiServerURL, taskId, taskSecret, logFile, cert string) (*Agent, error) {
	sh := &SignalHandler{}
	sh.makeChannels()

	// set up communicator with API server
	httpCommunicator, err := NewHTTPCommunicator(apiServerURL, taskId, taskSecret, cert, sh.communicatorChan)
	if err != nil {
		return nil, err
	}

	// set up logger to API server
	apiLogger := NewAPILogger(httpCommunicator)
	idleTimeoutWatcher := &TimeoutWatcher{duration: DefaultIdleTimeout, stop: sh.stopBackgroundChan}

	// set up timeout logger, local and API logger streams
	streamLogger, err := NewStreamLogger(idleTimeoutWatcher, apiLogger, logFile)
	if err != nil {
		return nil, err
	}
	httpCommunicator.Logger = streamLogger.Execution

	// set up the heartbeat ticker
	hbTicker := &HeartbeatTicker{
		MaxFailedHeartbeats: 10,
		SignalChan:          sh.heartbeatChan,
		TaskCommunicator:    httpCommunicator,
		Logger:              httpCommunicator.Logger,
		Interval:            DefaultHeartbeatInterval,
		stop:                sh.stopBackgroundChan,
	}

	// set up the system stats collector
	statsCollector := NewSimpleStatsCollector(
		streamLogger.System,
		DefaultStatsInterval,
		sh.stopBackgroundChan,
		"df -h",
		"${ps|ps}",
	)

	agt := &Agent{
		signalHandler:      sh,
		logger:             streamLogger,
		TaskCommunicator:   httpCommunicator,
		heartbeater:        hbTicker,
		statsCollector:     statsCollector,
		idleTimeoutWatcher: idleTimeoutWatcher,
		APILogger:          apiLogger,
		Registry:           plugin.NewSimpleRegistry(),
		KillChan:           make(chan bool),
		endChan:            make(chan *apimodels.TaskEndDetail, 1),
	}

	return agt, nil
}

func TestS3CopyPluginExecution(t *testing.T) {

	testConfig := evergreen.TestConfig()
	db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))

	testutil.ConfigureIntegrationTest(t, testConfig, "TestS3CopyPluginExecution")

	Convey("With a SimpleRegistry and test project file", t, func() {
		registry := plugin.NewSimpleRegistry()
		s3CopyPlugin := &S3CopyPlugin{}
		testutil.HandleTestingErr(registry.Register(s3CopyPlugin), t, "failed to register s3Copy plugin")
		testutil.HandleTestingErr(registry.Register(&s3Plugin.S3Plugin{}), t, "failed to register S3 plugin")
		testutil.HandleTestingErr(
			db.ClearCollections(model.PushlogCollection, version.Collection), t,
			"error clearing test collections")
		version := &version.Version{
			Id: "",
		}
		So(version.Insert(), ShouldBeNil)
		server, err := apiserver.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")

		httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)

		//server.InstallPlugin(s3CopyPlugin)

		taskConfig, err := plugintest.CreateTestConfig("testdata/plugin_s3_copy.yml", t)
		testutil.HandleTestingErr(err, t, "failed to create test config: %v", err)
		taskConfig.WorkDir = "."
		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agent.NewTestLogger(sliceAppender)

		taskConfig.Expansions.Update(map[string]string{
			"aws_key":    testConfig.Providers.AWS.Id,
			"aws_secret": testConfig.Providers.AWS.Secret,
		})

		Convey("the s3 copy command should execute successfully", func() {
			for _, task := range taskConfig.Project.Tasks {
				So(len(task.Commands), ShouldNotEqual, 0)
				for _, command := range task.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &agent.TaskJSONCommunicator{s3CopyPlugin.Name(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig,
						make(chan bool))
					So(err, ShouldBeNil)
				}
			}
		})
	})
}

func TestPluginFunctions(t *testing.T) {
	testConfig := evergreen.TestConfig()
	testutil.ConfigureIntegrationTest(t, testConfig, "TestPatchTask")
	Convey("With a SimpleRegistry", t, func() {
		Convey("with a project file containing functions", func() {
			registry := plugin.NewSimpleRegistry()
			err := registry.Register(&shell.ShellPlugin{})
			testutil.HandleTestingErr(err, t, "Couldn't register plugin")
			err = registry.Register(&expansions.ExpansionsPlugin{})
			testutil.HandleTestingErr(err, t, "Couldn't register plugin")

			testServer, err := apiserver.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
			testutil.HandleTestingErr(err, t, "Couldn't set up testing server")

			taskConfig, err := createTestConfig("testdata/plugin_project_functions.yml", t)
			testutil.HandleTestingErr(err, t, "failed to create test config: %v", err)

			Convey("all commands in project file should parse successfully", func() {
				for _, newTask := range taskConfig.Project.Tasks {
					for _, command := range newTask.Commands {
						pluginCmd, err := registry.GetCommands(command, taskConfig.Project.Functions)
						testutil.HandleTestingErr(err, t, "Got error getting plugin command: %v")
						So(pluginCmd, ShouldNotBeNil)
						So(err, ShouldBeNil)
					}
				}
			})

			httpCom, err := agent.NewHTTPCommunicator(testServer.URL, "mocktaskid", "mocktasksecret", "", nil)
			So(err, ShouldBeNil)
			So(httpCom, ShouldNotBeNil)

			Convey("all commands in test project should execute successfully", func() {
				sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
				logger := agent.NewTestLogger(sliceAppender)
				for _, newTask := range taskConfig.Project.Tasks {
					So(len(newTask.Commands), ShouldNotEqual, 0)
					for _, command := range newTask.Commands {
						pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
						testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
						So(pluginCmds, ShouldNotBeNil)
						So(err, ShouldBeNil)
						So(len(pluginCmds), ShouldEqual, 1)
						cmd := pluginCmds[0]
						pluginCom := &agent.TaskJSONCommunicator{cmd.Plugin(), httpCom}
						err = cmd.Execute(logger, pluginCom, taskConfig, make(chan bool))
						So(err, ShouldBeNil)
					}
				}
			})
		})
	})
}

// New creates a new agent to run a given task.
func New(apiServerURL, taskId, taskSecret, logFile, cert string) (*Agent, error) {
	sigChan := make(chan Signal, 1)

	// set up communicator with API server
	httpCommunicator, err := NewHTTPCommunicator(apiServerURL, taskId, taskSecret, cert, sigChan)
	if err != nil {
		return nil, err
	}

	// set up logger to API server
	apiLogger := NewAPILogger(httpCommunicator)
	idleTimeoutWatcher := &TimeoutWatcher{duration: DefaultIdleTimeout}

	// set up timeout logger, local and API logger streams
	streamLogger, err := NewStreamLogger(idleTimeoutWatcher, apiLogger, logFile)
	if err != nil {
		return nil, err
	}
	httpCommunicator.Logger = streamLogger.Execution

	// set up the heartbeat ticker
	hbTicker := &HeartbeatTicker{
		MaxFailedHeartbeats: 10,
		SignalChan:          sigChan,
		TaskCommunicator:    httpCommunicator,
		Logger:              httpCommunicator.Logger,
		Interval:            DefaultHeartbeatInterval,
	}

	// set up the system stats collector
	statsCollector := NewSimpleStatsCollector(
		streamLogger.System,
		DefaultStatsInterval,
		"df -h",
		"${ps|ps}",
	)

	agt := &Agent{
		logger:             streamLogger,
		TaskCommunicator:   httpCommunicator,
		heartbeater:        hbTicker,
		statsCollector:     statsCollector,
		idleTimeoutWatcher: idleTimeoutWatcher,
		APILogger:          apiLogger,
		signalChan:         sigChan,
		Registry:           plugin.NewSimpleRegistry(),
	}

	return agt, nil
}

func TestAttachResults(t *testing.T) {
	resetTasks(t)
	testConfig := evergreen.TestConfig()
	cwd := testutil.GetDirectoryOfFile()
	Convey("With attachResults plugin installed into plugin registry", t, func() {
		registry := plugin.NewSimpleRegistry()
		attachPlugin := &AttachPlugin{}
		err := registry.Register(attachPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")

		server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)
		configFile := filepath.Join(cwd, "testdata", "plugin_attach_results.yml")
		resultsLoc := filepath.Join(cwd, "testdata", "plugin_attach_results.json")
		taskConfig, err := plugintest.CreateTestConfig(configFile, t)
		testutil.HandleTestingErr(err, t, "failed to create test config: %v")
		taskConfig.WorkDir = "."
		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agentutil.NewTestLogger(sliceAppender)

		Convey("all commands in test project should execute successfully", func() {
			for _, projTask := range taskConfig.Project.Tasks {
				So(len(projTask.Commands), ShouldNotEqual, 0)
				for _, command := range projTask.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
					So(err, ShouldBeNil)
					testTask, err := task.FindOne(task.ById(httpCom.TaskId))
					testutil.HandleTestingErr(err, t, "Couldn't find task")
					So(testTask, ShouldNotBeNil)
					// ensure test results are exactly as expected
					// attempt to open the file
					reportFile, err := os.Open(resultsLoc)
					testutil.HandleTestingErr(err, t, "Couldn't open report file: '%v'", err)
					results := &task.TestResults{}
					err = util.ReadJSONInto(reportFile, results)
					testutil.HandleTestingErr(err, t, "Couldn't read report file: '%v'", err)
					testResults := *results
					So(testTask.TestResults, ShouldResemble, testResults.Results)
					testutil.HandleTestingErr(err, t, "Couldn't clean up test temp dir")
				}
			}
		})
	})
}

// Ensures there any plugin commands referenced in a project's configuration
// are specified in a valid format
func validatePluginCommands(project *model.Project) []ValidationError {
	errs := []ValidationError{}
	pluginRegistry := plugin.NewSimpleRegistry()

	// register the published plugins
	for _, pl := range plugin.Published {
		if err := pluginRegistry.Register(pl); err != nil {
			errs = append(errs,
				ValidationError{
					Message: fmt.Sprintf("failed to register plugin %v: %v", pl.Name(), err),
				},
			)
		}
	}

	// validate each function definition
	for funcName, commands := range project.Functions {
		valErrs := validateCommands("functions", project, pluginRegistry, commands.List())
		for _, err := range valErrs {
			errs = append(errs,
				ValidationError{
					Message: fmt.Sprintf("'%v' project's '%v' definition: %v",
						project.Identifier, funcName, err),
				},
			)
		}
	}

	if project.Pre != nil {
		// validate project pre section
		errs = append(errs, validateCommands("pre", project, pluginRegistry, project.Pre.List())...)
	}

	if project.Post != nil {
		// validate project post section
		errs = append(errs, validateCommands("post", project, pluginRegistry, project.Post.List())...)
	}

	if project.Timeout != nil {
		// validate project timeout section
		errs = append(errs, validateCommands("timeout", project, pluginRegistry, project.Timeout.List())...)
	}

	// validate project tasks section
	for _, task := range project.Tasks {
		errs = append(errs, validateCommands("tasks", project, pluginRegistry, task.Commands)...)
	}
	return errs
}

func TestPatchPlugin(t *testing.T) {
	cwd := testutil.GetDirectoryOfFile()
	testConfig := evergreen.TestConfig()
	db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))
	Convey("With patch plugin installed into plugin registry", t, func() {
		registry := plugin.NewSimpleRegistry()
		gitPlugin := &GitPlugin{}
		err := registry.Register(gitPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin %v")
		testutil.HandleTestingErr(db.Clear(version.Collection), t,
			"unable to clear versions collection")
		version := &version.Version{
			Id: "",
		}
		So(version.Insert(), ShouldBeNil)
		server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("testTaskId", "testTaskSecret", server.URL)

		//sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		sliceAppender := slogger.StdOutAppender()
		logger := agentutil.NewTestLogger(sliceAppender)

		Convey("all commands in test project should execute successfully", func() {
			taskConfig, err := plugintest.CreateTestConfig(filepath.Join(cwd, "testdata", "plugin_patch.yml"), t)
			testutil.HandleTestingErr(err, t, "could not create test config")

			taskConfig.Task.Requester = evergreen.PatchVersionRequester
			_, _, err = plugintest.SetupAPITestData("testTask", filepath.Join(cwd, "testdata", "testmodule.patch"), t)
			testutil.HandleTestingErr(err, t, "Couldn't set up test documents")

			for _, task := range taskConfig.Project.Tasks {
				So(len(task.Commands), ShouldNotEqual, 0)
				for _, command := range task.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
					So(err, ShouldBeNil)
				}
			}
		})
	})
}

func TestGotestPluginWithEnvironmentVariables(t *testing.T) {
	Convey("With gotest plugin installed into plugin registry", t, func() {
		reset(t)
		testConfig := evergreen.TestConfig()
		testutil.ConfigureIntegrationTest(t, testConfig, "TestGotestPluginWithEnvironmentVariables")
		registry := plugin.NewSimpleRegistry()
		testPlugin := &GotestPlugin{}
		err := registry.Register(testPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin %v")

		server, err := apiserver.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("testTaskId", "testTaskSecret", server.URL)

		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agent.NewTestLogger(sliceAppender)

		Convey("test command should get a copy of custom environment variables", func() {
			curWD, err := os.Getwd()
			testutil.HandleTestingErr(err, t, "Couldn't get working directory: %v")
			taskConfig, err := plugintest.CreateTestConfig("testdata/env.yml", t)
			// manually override working directory to the main repo, since this
			// is much easier than copying over the required testing dependencies
			// to a temporary directory
			testutil.HandleTestingErr(err, t, "Couldn't set up test config %v")
			taskConfig.WorkDir = curWD
			_, _, err = plugintest.SetupAPITestData("testTask", false, t)
			testutil.HandleTestingErr(err, t, "Couldn't set up test documents")

			for _, task := range taskConfig.Project.Tasks {
				So(len(task.Commands), ShouldNotEqual, 0)
				for _, command := range task.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &agent.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))

					So(err, ShouldBeNil)
				}
			}
		})
	})
}

func TestPluginExecution(t *testing.T) {
	Convey("With a SimpleRegistry and test project file", t, func() {
		registry := plugin.NewSimpleRegistry()

		plugins := []plugin.CommandPlugin{&MockPlugin{}, &expansions.ExpansionsPlugin{}, &shell.ShellPlugin{}}
		apiPlugins := []plugin.APIPlugin{&MockPlugin{}, &expansions.ExpansionsPlugin{}}
		for _, p := range plugins {
			err := registry.Register(p)
			testutil.HandleTestingErr(err, t, "failed to register plugin")
		}

		testServer, err := service.CreateTestServer(evergreen.TestConfig(), nil, apiPlugins, false)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")

		httpCom, err := comm.NewHTTPCommunicator(testServer.URL, "mocktaskid", "mocktasksecret", "", nil)
		So(err, ShouldBeNil)
		So(httpCom, ShouldNotBeNil)

		pluginConfigPath := filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "plugin_project.yml")
		taskConfig, err := createTestConfig(pluginConfigPath, t)
		testutil.HandleTestingErr(err, t, "failed to create test config: %v", err)
		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agentutil.NewTestLogger(sliceAppender)

		Convey("all commands in test project should execute successfully", func() {
			for _, newTask := range taskConfig.Project.Tasks {
				So(len(newTask.Commands), ShouldNotEqual, 0)
				for _, command := range newTask.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					for _, c := range pluginCmds {
						pluginCom := &comm.TaskJSONCommunicator{c.Plugin(), httpCom}
						err = c.Execute(logger, pluginCom, taskConfig, make(chan bool))
						So(err, ShouldBeNil)
					}
				}
			}
		})
	})
}

// runTest abstracts away common tests and setup between all attach xunit tests.
// It also takes as an argument a function which runs any additional tests desired.
func runTest(t *testing.T, configPath string, customTests func()) {
	resetTasks(t)
	testConfig := evergreen.TestConfig()
	Convey("With attachResults plugin installed into plugin registry", t, func() {
		registry := plugin.NewSimpleRegistry()
		attachPlugin := &AttachPlugin{}
		err := registry.Register(attachPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")

		server, err := apiserver.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)
		taskConfig, err := plugintest.CreateTestConfig(configPath, t)
		testutil.HandleTestingErr(err, t, "failed to create test config: %v")
		taskConfig.WorkDir = "."
		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agent.NewTestLogger(sliceAppender)

		Convey("all commands in test project should execute successfully", func() {
			for _, projTask := range taskConfig.Project.Tasks {
				So(len(projTask.Commands), ShouldNotEqual, 0)
				for _, command := range projTask.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &agent.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
					So(err, ShouldBeNil)
					testTask, err := task.FindOne(task.ById(httpCom.TaskId))
					testutil.HandleTestingErr(err, t, "Couldn't find task")
					So(testTask, ShouldNotBeNil)
				}
			}
			Convey("and the tests should be present in the db", customTests)
		})
	})
}

func TestIncKey(t *testing.T) {
	Convey("With keyval plugin installed", t, func() {
		err := db.Clear(keyval.KeyValCollection)
		util.HandleTestingErr(err, t, "Couldn't clear test collection: %v")
		registry := plugin.NewSimpleRegistry()
		kvPlugin := &keyval.KeyValPlugin{}
		err = registry.Register(kvPlugin)
		util.HandleTestingErr(err, t, "Couldn't register plugin: %v")

		server, err := apiserver.CreateTestServer(evergreen.TestConfig(), nil, []plugin.Plugin{kvPlugin}, false)
		httpCom := testutil.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)
		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agent.NewTestAgentLogger(sliceAppender)
		taskConfig, err := testutil.CreateTestConfig("testdata/plugin_keyval.yml", t)
		util.HandleTestingErr(err, t, "failed to create test config")

		Convey("Inc command should increment a key successfully", func() {
			for _, task := range taskConfig.Project.Tasks {
				So(len(task.Commands), ShouldNotEqual, 0)
				for _, command := range task.Commands {
					pluginCmds, err := registry.GetCommands(command, nil)
					util.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					for _, cmd := range pluginCmds {
						pluginCom := &agent.TaskJSONCommunicator{cmd.Plugin(), httpCom}
						err = cmd.Execute(logger, pluginCom, taskConfig, make(chan bool))
						So(err, ShouldBeNil)
					}
				}
				So(taskConfig.Expansions.Get("testkey"), ShouldEqual, "2")
				So(taskConfig.Expansions.Get("testkey_x"), ShouldEqual, "1")
			}
		})
	})
}

func TestGotestPluginOnPassingTests(t *testing.T) {
	SkipConvey("With gotest plugin installed into plugin registry", t, func() {
		reset(t)
		testConfig := evergreen.TestConfig()
		testutil.ConfigureIntegrationTest(t, testConfig, "TestGotestPluginOnPassingTests")
		registry := plugin.NewSimpleRegistry()
		testPlugin := &GotestPlugin{}
		err := registry.Register(testPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin %v")

		server, err := apiserver.CreateTestServer(evergreen.TestConfig(), nil, plugin.APIPlugins, true)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("testTaskId", "testTaskSecret", server.URL)

		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agent.NewTestLogger(sliceAppender)

		Convey("all commands in test project should execute successfully", func() {
			curWD, err := os.Getwd()
			testutil.HandleTestingErr(err, t, "Couldn't get working directory: %v")
			taskConfig, err := plugintest.CreateTestConfig("testdata/good.yml", t)
			// manually override working directory to the main repo, since this
			// is much easier than copying over the required testing dependencies
			// to a temporary directory
			testutil.HandleTestingErr(err, t, "Couldn't set up test config %v")
			taskConfig.WorkDir = curWD
			task, _, err := plugintest.SetupAPITestData("testTask", false, t)
			testutil.HandleTestingErr(err, t, "Couldn't set up test documents")

			for _, task := range taskConfig.Project.Tasks {
				So(len(task.Commands), ShouldNotEqual, 0)
				for _, command := range task.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &agent.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))

					So(err, ShouldBeNil)
				}
			}

			Convey("and the tests in the task should be updated", func() {
				updatedTask, err := model.FindTask(task.Id)
				So(err, ShouldBeNil)
				So(updatedTask, ShouldNotBeNil)
				So(len(updatedTask.TestResults), ShouldEqual, 2)
				So(updatedTask.TestResults[0].Status, ShouldEqual, "pass")
				So(updatedTask.TestResults[1].Status, ShouldEqual, "pass")
				So(updatedTask.TestResults[0].TestFile, ShouldEqual, "TestPass01")
				So(updatedTask.TestResults[1].TestFile, ShouldEqual, "TestPass02")
				So(updatedTask.TestResults[0].StartTime, ShouldBeLessThan,
					updatedTask.TestResults[0].EndTime)
				So(updatedTask.TestResults[1].StartTime, ShouldBeLessThan,
					updatedTask.TestResults[1].EndTime)

				Convey("with relevant logs present in the DB as well", func() {
					log, err := model.FindOneTestLog("0_goodpkg", "testTaskId", 0)
					So(log, ShouldNotBeNil)
					So(err, ShouldBeNil)
					So(log.Lines[0], ShouldContainSubstring, "TestPass01")
				})

			})
		})
	})
}

func TestAttachRawResults(t *testing.T) {
	resetTasks(t)
	testConfig := evergreen.TestConfig()
	cwd := testutil.GetDirectoryOfFile()
	Convey("With attachResults plugin installed into plugin registry", t, func() {
		registry := plugin.NewSimpleRegistry()
		attachPlugin := &AttachPlugin{}
		err := registry.Register(attachPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")

		server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)
		configFile := filepath.Join(cwd, "testdata", "plugin_attach_results_raw.yml")
		resultsLoc := filepath.Join(cwd, "testdata", "plugin_attach_results_raw.json")
		taskConfig, err := plugintest.CreateTestConfig(configFile, t)
		testutil.HandleTestingErr(err, t, "failed to create test config: %v")
		taskConfig.WorkDir = "."
		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agentutil.NewTestLogger(sliceAppender)

		Convey("when attaching a raw log ", func() {
			for _, projTask := range taskConfig.Project.Tasks {
				So(len(projTask.Commands), ShouldNotEqual, 0)
				for _, command := range projTask.Commands {

					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					// create a plugin communicator
					pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
					So(err, ShouldBeNil)
					Convey("when retrieving task", func() {
						// fetch the task
						testTask, err := task.FindOne(task.ById(httpCom.TaskId))
						testutil.HandleTestingErr(err, t, "Couldn't find task")
						So(testTask, ShouldNotBeNil)

						Convey("test results should match and raw log should be in appropriate collection", func() {

							reportFile, err := os.Open(resultsLoc)
							testutil.HandleTestingErr(err, t, "Couldn't open report file: '%v'", err)
							results := &task.TestResults{}
							err = util.ReadJSONInto(reportFile, results)
							testutil.HandleTestingErr(err, t, "Couldn't read report file: '%v'", err)

							testResults := *results
							So(len(testResults.Results), ShouldEqual, 3)
							So(len(testTask.TestResults), ShouldEqual, 3)
							firstResult := testTask.TestResults[0]
							So(firstResult.LogRaw, ShouldEqual, "")
							So(firstResult.LogId, ShouldNotEqual, "")

							testLog, err := model.FindOneTestLogById(firstResult.LogId)
							So(err, ShouldBeNil)
							So(testLog.Lines[0], ShouldEqual, testResults.Results[0].LogRaw)

							Convey("both URL and raw log should be stored appropriately if both exist", func() {
								urlResult := testTask.TestResults[2]
								So(urlResult.LogRaw, ShouldEqual, "")
								So(urlResult.URL, ShouldNotEqual, "")
								So(urlResult.LogId, ShouldNotEqual, "")

								testLog, err := model.FindOneTestLogById(urlResult.LogId)
								So(err, ShouldBeNil)
								So(testLog.Lines[0], ShouldEqual, testResults.Results[2].LogRaw)
							})
						})
					})

				}
			}
		})
	})
}

func TestAttachTaskFilesPlugin(t *testing.T) {
	testConfig := evergreen.TestConfig()
	Convey("With attach plugin installed into plugin registry", t, func() {
		registry := plugin.NewSimpleRegistry()
		attachPlugin := &AttachPlugin{}
		err := registry.Register(attachPlugin)
		testutil.HandleTestingErr(err, t, "Couldn't register plugin %v")

		server, err := apiserver.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
		testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
		httpCom := plugintest.TestAgentCommunicator("testTaskId", "testTaskSecret", server.URL)

		sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
		logger := agent.NewTestLogger(sliceAppender)
		Convey("all commands in test project should execute successfully", func() {
			taskConfig, _ := plugintest.CreateTestConfig("testdata/plugin_attach_files.yml", t)
			_, _, err = plugintest.SetupAPITestData("testTask", true, t)
			testutil.HandleTestingErr(err, t, "Couldn't set up test documents")

			for _, task := range taskConfig.Project.Tasks {
				So(len(task.Commands), ShouldNotEqual, 0)
				for _, command := range task.Commands {
					pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
					testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
					So(pluginCmds, ShouldNotBeNil)
					So(err, ShouldBeNil)
					pluginCom := &agent.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
					err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
					So(err, ShouldBeNil)
				}
			}

			Convey("and these file entry fields should exist in the db:", func() {
				entry, err := artifact.FindOne(artifact.ByTaskId("testTaskId"))
				So(err, ShouldBeNil)
				So(entry, ShouldNotBeNil)
				So(entry.TaskDisplayName, ShouldEqual, "testTask")
				So(len(entry.Files), ShouldEqual, 5)

				var regular artifact.File
				var expansion artifact.File
				var overwritten artifact.File

				for _, file := range entry.Files {
					switch file.Name {
					case "file1":
						expansion = file
					case "file2":
						overwritten = file
					case "file3":
						regular = file
					}
				}
				Convey("- regular link", func() {
					So(regular, ShouldResemble,
						artifact.File{"file3", "http://kyle.diamonds", ""})
				})

				Convey("- link with expansion", func() {
					So(expansion, ShouldResemble,
						artifact.File{"file1", "i am a FILE!", ""})
				})

				Convey("- link that is overwritten", func() {
					So(overwritten, ShouldResemble,
						artifact.File{"file2", "replaced!", ""})
				})
			})
		})
	})
}

func TestAttachFilesApi(t *testing.T) {
	testConfig := evergreen.TestConfig()
	Convey("With a running api server and installed api hook", t, func() {
		reset(t)
		taskConfig, _ := plugintest.CreateTestConfig("testdata/plugin_attach_files.yml", t)
		registry := plugin.NewSimpleRegistry()
		attachPlugin := &AttachPlugin{}
		err := regi