func (s *suite) TestClosedChangesChannel(c *gc.C) {
	s.runDirtyTest(c, func(w worker.Worker, backend *mockBackend) {
		backend.sendModelChange("uuid1", "uuid2")
		workers := s.waitWorkers(c, 2)

		close(backend.envWatcher.changes)
		err := workertest.CheckKilled(c, w)
		c.Check(err, gc.ErrorMatches, "changes stopped")
		for _, worker := range workers {
			workertest.CheckKilled(c, worker)
		}
		s.assertNoWorkers(c)
	})
}

func (*WorkerSuite) TestStopKills(c *gc.C) {
	w := workertest.NewErrorWorker(nil)
	defer workertest.CleanKill(c, w)

	worker.Stop(w)
	workertest.CheckKilled(c, w)
}

func (s *Suite) TestSUCCESSMinionWaitTimeout(c *gc.C) {
	// The SUCCESS phase is special in that even if some minions fail
	// to report the migration should continue. There's no turning
	// back from SUCCESS.
	s.facade.queueStatus(s.makeStatus(coremigration.SUCCESS))

	worker, err := migrationmaster.New(s.config)
	c.Assert(err, jc.ErrorIsNil)
	defer workertest.DirtyKill(c, worker)

	select {
	case <-s.clock.Alarms():
	case <-time.After(coretesting.LongWait):
		c.Fatal("timed out waiting for clock.After call")
	}

	// Move time ahead in order to trigger timeout.
	s.clock.Advance(15 * time.Minute)

	err = workertest.CheckKilled(c, worker)
	c.Assert(err, gc.Equals, migrationmaster.ErrMigrated)

	s.stub.CheckCalls(c, joinCalls(
		watchStatusLockdownCalls,
		[]jujutesting.StubCall{
			{"facade.WatchMinionReports", nil},
			{"facade.SetPhase", []interface{}{coremigration.LOGTRANSFER}},
			{"facade.SetPhase", []interface{}{coremigration.REAP}},
			{"facade.Reap", nil},
			{"facade.SetPhase", []interface{}{coremigration.DONE}},
		},
	))
}

func (s *Suite) TestClosedWatcherChannel(c *gc.C) {
	close(s.client.watcher.changes)
	w, err := migrationminion.New(s.config)
	c.Assert(err, jc.ErrorIsNil)
	err = workertest.CheckKilled(c, w)
	c.Check(err, gc.ErrorMatches, "watcher channel closed")
}

func (*WorkerSuite) TestStopWaits(c *gc.C) {
	w := workertest.NewForeverWorker(nil)
	defer workertest.CheckKilled(c, w)
	defer w.ReallyKill()

	done := make(chan struct{})
	go func() {
		defer close(done)
		worker.Stop(w)
	}()

	select {
	case <-time.After(coretesting.ShortWait):
	case <-done:
		c.Fatalf("Stop returned early")
	}

	w.ReallyKill()

	select {
	case <-done:
	case <-time.After(coretesting.LongWait):
		c.Fatalf("Stop never returned")
	}
}

// TestWorstError starts an engine with two manifolds that always error
// with fatal errors. We test that the most important error is the one
// returned by the engine.
//
// This test uses manifolds whose workers ignore kill requests. We want
// this (dangerous!) behaviour so that we don't race over which fatal
// error is seen by the engine first.
func (s *EngineSuite) TestWorstError(c *gc.C) {
	worstErr := errors.New("awful error")
	callCount := 0
	s.fix.worstError = func(err1, err2 error) error {
		callCount++
		return worstErr
	}
	s.fix.isFatal = alwaysFatal
	s.fix.dirty = true
	s.fix.run(c, func(engine dependency.Engine) {

		mh1 := newErrorIgnoringManifoldHarness()
		err := engine.Install("task", mh1.Manifold())
		c.Assert(err, jc.ErrorIsNil)
		mh1.AssertOneStart(c)

		mh2 := newErrorIgnoringManifoldHarness()
		err = engine.Install("another task", mh2.Manifold())
		c.Assert(err, jc.ErrorIsNil)
		mh2.AssertOneStart(c)

		mh1.InjectError(c, errors.New("ping"))
		mh2.InjectError(c, errors.New("pong"))

		err = workertest.CheckKilled(c, engine)
		c.Check(errors.Cause(err), gc.Equals, worstErr)
		c.Check(callCount, gc.Equals, 2)
	})
}

// cleanup checks that every expected worker has already been stopped by
// the SUT. (i.e.: don't set up more workers than your fixture needs).
func (wl *workerList) cleanup(c *gc.C) {
	for _, w := range wl.workers {
		if w != nil {
			workertest.CheckKilled(c, w)
		}
	}
}

func (s *UndertakerSuite) TestProcessDyingModelErrorRetried(c *gc.C) {
	s.fix.errors = []error{
		nil, // ModelInfo
		nil, // SetStatus
		nil, // WatchModelResources,
		errors.New("meh, will retry"),  // ProcessDyingModel,
		errors.New("will retry again"), // ProcessDyingModel,
		nil, // ProcessDyingModel,
		nil, // SetStatus
		nil, // Destroy,
		nil, // RemoveModel
	}
	stub := s.fix.run(c, func(w worker.Worker) {
		workertest.CheckKilled(c, w)
	})
	stub.CheckCallNames(c,
		"ModelInfo",
		"SetStatus",
		"WatchModelResources",
		"ProcessDyingModel",
		"ProcessDyingModel",
		"ProcessDyingModel",
		"SetStatus",
		"Destroy",
		"RemoveModel",
	)
}

func (s *Suite) TestImportFailure(c *gc.C) {
	masterClient := newStubMasterClient(s.stub)
	worker, err := migrationmaster.New(migrationmaster.Config{
		Facade: masterClient,
		Guard:  newStubGuard(s.stub),
	})
	c.Assert(err, jc.ErrorIsNil)
	s.connection.importErr = errors.New("boom")
	s.triggerMigration(masterClient)

	err = workertest.CheckKilled(c, worker)
	c.Assert(err, gc.Equals, migrationmaster.ErrDoneForNow)

	s.stub.CheckCalls(c, []jujutesting.StubCall{
		{"masterClient.Watch", nil},
		{"masterClient.GetMigrationStatus", nil},
		{"guard.Lockdown", nil},
		{"masterClient.SetPhase", []interface{}{migration.READONLY}},
		{"masterClient.SetPhase", []interface{}{migration.PRECHECK}},
		{"masterClient.SetPhase", []interface{}{migration.IMPORT}},
		{"masterClient.Export", nil},
		apiOpenCall,
		importCall,
		connCloseCall,
		{"masterClient.SetPhase", []interface{}{migration.ABORT}},
		apiOpenCall,
		abortCall,
		connCloseCall,
		{"masterClient.SetPhase", []interface{}{migration.ABORTDONE}},
	})
}

func (s *UndertakerSuite) TestAlreadyDeadRemoves(c *gc.C) {
	s.fix.info.Result.Life = "dead"
	stub := s.fix.run(c, func(w worker.Worker) {
		workertest.CheckKilled(c, w)
	})
	stub.CheckCallNames(c, "ModelInfo", "SetStatus", "Destroy", "RemoveModel")
}

func (s *Suite) TestWatchFailure(c *gc.C) {
	s.client.watchErr = errors.New("boom")
	w, err := migrationminion.New(s.config)
	c.Assert(err, jc.ErrorIsNil)
	err = workertest.CheckKilled(c, w)
	c.Check(err, gc.ErrorMatches, "setting up watcher: boom")
}

func (s *EngineSuite) TestIsFatal(c *gc.C) {
	fatalErr := errors.New("KABOOM")
	s.fix.isFatal = isFatalIf(fatalErr)
	s.fix.dirty = true
	s.fix.run(c, func(engine dependency.Engine) {

		// Start two independent workers.
		mh1 := newManifoldHarness()
		err := engine.Install("some-task", mh1.Manifold())
		c.Assert(err, jc.ErrorIsNil)
		mh1.AssertOneStart(c)
		mh2 := newManifoldHarness()
		err = engine.Install("other-task", mh2.Manifold())
		c.Assert(err, jc.ErrorIsNil)
		mh2.AssertOneStart(c)

		// Bounce one worker with Just Some Error; check that worker bounces.
		mh1.InjectError(c, errors.New("splort"))
		mh1.AssertOneStart(c)
		mh2.AssertNoStart(c)

		// Bounce another worker with the fatal error; check the engine exits with
		// the right error.
		mh2.InjectError(c, fatalErr)
		mh1.AssertNoStart(c)
		mh2.AssertNoStart(c)
		err = workertest.CheckKilled(c, engine)
		c.Assert(err, gc.Equals, fatalErr)
	})
}

func (*WorkerSuite) TestWatchErrorNonEmptyRunningActions(c *gc.C) {
	stub := &testing.Stub{}
	stub.SetErrors(nil, errors.New("ignored"), errors.New("kuso"))
	facade := &mockFacade{
		stub:           stub,
		runningActions: fakeRunningActions,
	}
	config := machineactions.WorkerConfig{
		Facade:       facade,
		MachineTag:   fakeTag,
		HandleAction: mockHandleAction(stub),
	}
	worker, err := machineactions.NewMachineActionsWorker(config)
	c.Assert(err, jc.ErrorIsNil)
	err = workertest.CheckKilled(c, worker)
	c.Check(err, gc.ErrorMatches, "kuso")

	stub.CheckCalls(c, []testing.StubCall{{
		FuncName: "RunningActions",
		Args:     []interface{}{fakeTag},
	}, {
		FuncName: "ActionFinish",
		Args:     []interface{}{thirdActionTag, params.ActionFailed, "action cancelled"},
	}, {
		FuncName: "WatchActionNotifications",
		Args:     []interface{}{fakeTag},
	}})
}

func (s *Suite) TestReportKeysError(c *gc.C) {
	s.facade.reportErr = errors.New("blam")
	w, err := hostkeyreporter.New(s.config)
	c.Assert(err, jc.ErrorIsNil)
	err = workertest.CheckKilled(c, w)
	c.Check(err, gc.ErrorMatches, "blam")
}

func (*ResumerSuite) TestImmediateFailure(c *gc.C) {
	fix := newFixture(errors.New("zap"))
	stub := fix.Run(c, func(_ *testing.Clock, worker *resumer.Resumer) {
		err := workertest.CheckKilled(c, worker)
		c.Check(err, gc.ErrorMatches, "cannot resume transactions: zap")
	})
	stub.CheckCallNames(c, "ResumeTransactions")
}

func (s *Suite) TestCheckKilledTimeout(c *gc.C) {
	w := workertest.NewErrorWorker(nil)
	defer workertest.CleanKill(c, w)

	err := workertest.CheckKilled(c, w)
	s.CheckFailed(c)
	c.Check(err, gc.ErrorMatches, "workertest: worker not stopping")
}

func (context *context) checkCleanedUp() {
	context.c.Logf("checking no active current pinger")
	context.mu.Lock()
	defer context.mu.Unlock()
	if context.current != nil {
		workertest.CheckKilled(context.c, context.current)
	}
}

func (s *Suite) TestCheckKilledSuccess(c *gc.C) {
	expect := errors.New("snifplog")
	w := workertest.NewErrorWorker(expect)
	defer workertest.DirtyKill(c, w)

	w.Kill()
	err := workertest.CheckKilled(c, w)
	c.Check(err, gc.Equals, expect)
}

func (s *UndertakerSuite) TestAlreadyDeadTimeMissingFinishes(c *gc.C) {
	s.fix.info.Result.Life = "dead"
	stub := s.fix.run(c, func(w worker.Worker, clock *coretesting.Clock) {
		waitAlarm(c, clock)
		clock.Advance(RIPTime)
		workertest.CheckKilled(c, w)
	})
	stub.CheckCallNames(c, "ModelInfo", "Destroy", "RemoveModel")
}

func (s *undertakerSuite) TestErrorGettingRemovals(c *gc.C) {
	api := s.makeAPIWithWatcher()
	api.SetErrors(nil, errors.New("explodo"))
	w, err := machineundertaker.NewWorker(api, &fakeEnviron{})
	c.Assert(err, jc.ErrorIsNil)
	err = workertest.CheckKilled(c, w)
	c.Check(err, gc.ErrorMatches, "explodo")
	api.CheckCallNames(c, "WatchMachineRemovals", "AllMachineRemovals")
}