func (s *pingerSuite) calculatePingTimeout(c *gc.C) time.Duration {
	// Try opening an API connection a few times and take the max
	// delay among the attempts.
	attempt := utils.AttemptStrategy{
		Delay: coretesting.ShortWait,
		Min:   3,
	}
	var maxTimeout time.Duration
	for a := attempt.Start(); a.Next(); {
		openStart := time.Now()
		st, _ := s.OpenAPIAsNewMachine(c)
		err := st.Ping()
		if c.Check(err, jc.ErrorIsNil) {
			openDelay := time.Since(openStart)
			c.Logf("API open and initial ping took %v", openDelay)
			if maxTimeout < openDelay {
				maxTimeout = openDelay
			}
		}
		if st != nil {
			c.Check(st.Close(), jc.ErrorIsNil)
		}
	}
	if !c.Failed() && maxTimeout > 0 {
		return maxTimeout
	}
	c.Fatalf("cannot calculate ping timeout")
	return 0
}

// waitForAgentInitialisation polls the bootstrapped state server with a read-only
// command which will fail until the state server is fully initialised.
// TODO(wallyworld) - add a bespoke command to maybe the admin facade for this purpose.
func (c *BootstrapCommand) waitForAgentInitialisation(ctx *cmd.Context) (err error) {
	attempts := utils.AttemptStrategy{
		Min:   bootstrapReadyPollCount,
		Delay: bootstrapReadyPollDelay,
	}
	var client block.BlockListAPI
	for attempt := attempts.Start(); attempt.Next(); {
		client, err = blockAPI(&c.EnvCommandBase)
		if err != nil {
			return err
		}
		_, err = client.List()
		client.Close()
		if err == nil {
			ctx.Infof("Bootstrap complete")
			return nil
		}
		if strings.Contains(err.Error(), apiserver.UpgradeInProgressError.Error()) {
			ctx.Infof("Waiting for API to become available")
			continue
		}
		return err
	}
	return err
}

func createWebsocketDialer(cfg *websocket.Config, opts DialOpts) func(<-chan struct{}) (io.Closer, error) {
	openAttempt := utils.AttemptStrategy{
		Total: opts.Timeout,
		Delay: opts.RetryDelay,
	}
	return func(stop <-chan struct{}) (io.Closer, error) {
		for a := openAttempt.Start(); a.Next(); {
			select {
			case <-stop:
				return nil, parallel.ErrStopped
			default:
			}
			logger.Infof("dialing %q", cfg.Location)
			conn, err := websocket.DialConfig(cfg)
			if err == nil {
				return conn, nil
			}
			if a.HasNext() {
				logger.Debugf("error dialing %q, will retry: %v", cfg.Location, err)
			} else {
				logger.Infof("error dialing %q: %v", cfg.Location, err)
				return nil, errors.Errorf("unable to connect to %q", cfg.Location)
			}
		}
		panic("unreachable")
	}
}

// newWebsocketDialer0 returns a function that dials the websocket represented
// by the given configuration with the given dial options, suitable for passing
// to utils/parallel.Try.Start.
func newWebsocketDialer(cfg *websocket.Config, opts DialOpts) func(<-chan struct{}) (io.Closer, error) {
	// TODO(katco): 2016-08-09: lp:1611427
	openAttempt := utils.AttemptStrategy{
		Total: opts.Timeout,
		Delay: opts.RetryDelay,
	}
	return func(stop <-chan struct{}) (io.Closer, error) {
		for a := openAttempt.Start(); a.Next(); {
			select {
			case <-stop:
				return nil, parallel.ErrStopped
			default:
			}
			logger.Infof("dialing %q", cfg.Location)
			conn, err := opts.DialWebsocket(cfg)
			if err == nil {
				return conn, nil
			}
			if !a.HasNext() || isX509Error(err) {
				// We won't reconnect when there's an X509 error
				// because we're not going to succeed if we retry
				// in that case.
				logger.Infof("error dialing %q: %v", cfg.Location, err)
				return nil, errors.Annotatef(err, "unable to connect to API")
			}
		}
		panic("unreachable")
	}
}

// waitOperation waits for the provided operation to reach the "done"
// status. It follows the given attempt strategy (e.g. wait time between
// attempts) and may time out.
func (rc *rawConn) waitOperation(projectID string, op *compute.Operation, attempts utils.AttemptStrategy) error {
	started := time.Now()
	logger.Infof("GCE operation %q, waiting...", op.Name)
	for a := attempts.Start(); a.Next(); {
		if op.Status == StatusDone {
			break
		}

		var err error
		op, err = rc.checkOperation(projectID, op)
		if err != nil {
			return errors.Trace(err)
		}
	}
	if op.Status != StatusDone {
		err := errors.Errorf("timed out after %d seconds", time.Now().Sub(started)/time.Second)
		return waitError{op, err}
	}
	if op.Error != nil {
		for _, err := range op.Error.Errors {
			logger.Errorf("GCE operation error: (%s) %s", err.Code, err.Message)
		}
		return waitError{op, nil}
	}

	logger.Infof("GCE operation %q finished", op.Name)
	return nil
}

func checkFileHasContents(c *gc.C, stor storage.StorageReader, name string, contents []byte, attempt utils.AttemptStrategy) {
	r, err := storage.GetWithRetry(stor, name, attempt)
	c.Assert(err, gc.IsNil)
	c.Check(r, gc.NotNil)
	defer r.Close()

	data, err := ioutil.ReadAll(r)
	c.Check(err, gc.IsNil)
	c.Check(data, gc.DeepEquals, contents)

	url, err := stor.URL(name)
	c.Assert(err, gc.IsNil)

	var resp *http.Response
	for a := attempt.Start(); a.Next(); {
		resp, err = utils.GetValidatingHTTPClient().Get(url)
		c.Assert(err, gc.IsNil)
		if resp.StatusCode != 404 {
			break
		}
		c.Logf("get retrying after earlier get succeeded. *sigh*.")
	}
	c.Assert(err, gc.IsNil)
	data, err = ioutil.ReadAll(resp.Body)
	c.Assert(err, gc.IsNil)
	defer resp.Body.Close()
	c.Assert(resp.StatusCode, gc.Equals, 200, gc.Commentf("error response: %s", data))
	c.Check(data, gc.DeepEquals, contents)
}

func (c *restoreCommand) Run(ctx *cmd.Context) error {
	if c.showDescription {
		fmt.Fprintf(ctx.Stdout, "%s\n", c.Info().Purpose)
		return nil
	}
	if err := c.Log.Start(ctx); err != nil {
		return err
	}
	agentConf, err := extractConfig(c.backupFile)
	if err != nil {
		return errors.Annotate(err, "cannot extract configuration from backup file")
	}
	progress("extracted credentials from backup file")
	store, err := configstore.Default()
	if err != nil {
		return err
	}
	cfg, err := c.Config(store)
	if err != nil {
		return err
	}
	env, err := rebootstrap(cfg, ctx, c.Constraints)
	if err != nil {
		return errors.Annotate(err, "cannot re-bootstrap environment")
	}
	progress("connecting to newly bootstrapped instance")
	var apiState *api.State
	// The state server backend may not be ready to accept logins so we retry.
	// We'll do up to 8 retries over 2 minutes to give the server time to come up.
	// Typically we expect only 1 retry will be needed.
	attempt := utils.AttemptStrategy{Delay: 15 * time.Second, Min: 8}
	for a := attempt.Start(); a.Next(); {
		apiState, err = juju.NewAPIState(env, api.DefaultDialOpts())
		if err == nil || errors.Cause(err).Error() != "EOF" {
			break
		}
		progress("bootstrapped instance not ready - attempting to redial")
	}
	if err != nil {
		return errors.Annotate(err, "cannot connect to bootstrap instance")
	}
	progress("restoring bootstrap machine")
	machine0Addr, err := restoreBootstrapMachine(apiState, c.backupFile, agentConf)
	if err != nil {
		return errors.Annotate(err, "cannot restore bootstrap machine")
	}
	progress("restored bootstrap machine")

	apiState, err = juju.NewAPIState(env, api.DefaultDialOpts())
	progress("opening state")
	if err != nil {
		return errors.Annotate(err, "cannot connect to api server")
	}
	progress("updating all machines")
	if err := updateAllMachines(apiState, machine0Addr); err != nil {
		return errors.Annotate(err, "cannot update machines")
	}
	return nil
}

// GetWithRetry gets the named file from stor using the specified attempt strategy.
func GetWithRetry(stor StorageReader, name string, attempt utils.AttemptStrategy) (r io.ReadCloser, err error) {
	for a := attempt.Start(); a.Next(); {
		r, err = stor.Get(name)
		if err == nil || !stor.ShouldRetry(err) {
			break
		}
	}
	return r, err
}

// ListWithRetry lists the files matching prefix from stor using the specified attempt strategy.
func ListWithRetry(stor StorageReader, prefix string, attempt utils.AttemptStrategy) (list []string, err error) {
	for a := attempt.Start(); a.Next(); {
		list, err = stor.List(prefix)
		if err == nil || !stor.ShouldRetry(err) {
			break
		}
	}
	return list, err
}

// WaitForAgentInitialisation polls the bootstrapped controller with a read-only
// command which will fail until the controller is fully initialised.
// TODO(wallyworld) - add a bespoke command to maybe the admin facade for this purpose.
func WaitForAgentInitialisation(ctx *cmd.Context, c *modelcmd.ModelCommandBase, controllerName, hostedModelName string) error {
	// TODO(katco): 2016-08-09: lp:1611427
	attempts := utils.AttemptStrategy{
		Min:   bootstrapReadyPollCount,
		Delay: bootstrapReadyPollDelay,
	}
	var (
		apiAttempts int
		err         error
	)

	// Make a best effort to find the new controller address so we can print it.
	addressInfo := ""
	controller, err := c.ClientStore().ControllerByName(controllerName)
	if err == nil && len(controller.APIEndpoints) > 0 {
		addr, err := network.ParseHostPort(controller.APIEndpoints[0])
		if err == nil {
			addressInfo = fmt.Sprintf(" at %s", addr.Address.Value)
		}
	}

	ctx.Infof("Contacting Juju controller%s to verify accessibility...", addressInfo)
	apiAttempts = 1
	for attempt := attempts.Start(); attempt.Next(); apiAttempts++ {
		err = tryAPI(c)
		if err == nil {
			ctx.Infof("Bootstrap complete, %q controller now available.", controllerName)
			ctx.Infof("Controller machines are in the %q model.", bootstrap.ControllerModelName)
			ctx.Infof("Initial model %q added.", hostedModelName)
			break
		}
		// As the API server is coming up, it goes through a number of steps.
		// Initially the upgrade steps run, but the api server allows some
		// calls to be processed during the upgrade, but not the list blocks.
		// Logins are also blocked during space discovery.
		// It is also possible that the underlying database causes connections
		// to be dropped as it is initialising, or reconfiguring. These can
		// lead to EOF or "connection is shut down" error messages. We skip
		// these too, hoping that things come back up before the end of the
		// retry poll count.
		errorMessage := errors.Cause(err).Error()
		switch {
		case errors.Cause(err) == io.EOF,
			strings.HasSuffix(errorMessage, "connection is shut down"),
			strings.HasSuffix(errorMessage, "no api connection available"),
			strings.Contains(errorMessage, "spaces are still being discovered"):
			ctx.Verbosef("Still waiting for API to become available")
			continue
		case params.ErrCode(err) == params.CodeUpgradeInProgress:
			ctx.Verbosef("Still waiting for API to become available: %v", err)
			continue
		}
		break
	}
	return errors.Annotatef(err, "unable to contact api server after %d attempts", apiAttempts)
}

func (s *pingerSuite) TestAgentConnectionDelaysShutdownWithPing(c *gc.C) {
	// To negate the effects of an underpowered or heavily loaded
	// machine running this test, tune the shortTimeout based on the
	// maximum duration it takes to open an API connection.
	shortTimeout := s.calculatePingTimeout(c)
	attemptDelay := shortTimeout / 4

	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(shortTimeout))

	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	defer st.Close()

	// As long as we don't wait too long, the connection stays open
	attempt := utils.AttemptStrategy{
		Min:   10,
		Delay: attemptDelay,
	}
	testStart := time.Now()
	c.Logf(
		"pinging %d times with %v delay, ping timeout %v, starting at %v",
		attempt.Min, attempt.Delay, shortTimeout, testStart,
	)
	var lastLoop time.Time
	for a := attempt.Start(); a.Next(); {
		testNow := time.Now()
		loopDelta := testNow.Sub(lastLoop)
		if lastLoop.IsZero() {
			loopDelta = 0
		}
		c.Logf("duration since last ping: %v", loopDelta)
		err = st.Ping()
		if !c.Check(
			err, jc.ErrorIsNil,
			gc.Commentf(
				"ping timeout exceeded at %v (%v since the test start)",
				testNow, testNow.Sub(testStart),
			),
		) {
			c.Check(err, gc.ErrorMatches, "connection is shut down")
			return
		}
		lastLoop = time.Now()
	}

	// However, once we stop pinging for too long, the connection dies
	time.Sleep(2 * shortTimeout) // Exceed the timeout.
	err = st.Ping()
	c.Assert(err, gc.ErrorMatches, "connection is shut down")
}

func checkConnectionDies(c *gc.C, conn api.Connection) {
	attempt := utils.AttemptStrategy{
		Total: coretesting.LongWait,
		Delay: coretesting.ShortWait,
	}
	for a := attempt.Start(); a.Next(); {
		err := pingConn(conn)
		if err != nil {
			c.Assert(err, gc.ErrorMatches, "connection is shut down")
			return
		}
	}
	c.Fatal("connection didn't get shut down")
}

func attemptLoop(c *gc.C, strategy utils.AttemptStrategy, desc string, f func() error) {
	var err error
	start := time.Now()
	attemptCount := 0
	for attempt := strategy.Start(); attempt.Next(); {
		attemptCount += 1
		if err = f(); err == nil || !attempt.HasNext() {
			break
		}
		c.Logf("%s failed: %v", desc, err)
	}
	c.Logf("%s: %d attempts in %s", desc, attemptCount, time.Since(start))
	c.Assert(err, gc.IsNil)
}

// networkOperationWithRetries calls the supplied function and if it returns a
// network error which is temporary, will retry a number of times before giving up.
func networkOperationWithRetries(strategy utils.AttemptStrategy, networkOp func() error, description string) func() error {
	return func() error {
		for a := strategy.Start(); ; {
			a.Next()
			err := networkOp()
			if !a.HasNext() || err == nil {
				return errors.Trace(err)
			}
			if networkErr, ok := errors.Cause(err).(net.Error); !ok || !networkErr.Temporary() {
				return errors.Trace(err)
			}
			logger.Debugf("%q error, will retry: %v", description, err)
		}
	}
}

func (v *ebsVolumeSource) waitVolumeCreated(volumeId string) (*ec2.Volume, error) {
	var attempt = utils.AttemptStrategy{
		Total: 5 * time.Second,
		Delay: 200 * time.Millisecond,
	}
	for a := attempt.Start(); a.Next(); {
		volume, err := v.describeVolume(volumeId)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if volume.Status != volumeStatusCreating {
			return volume, nil
		}
	}
	return nil, errors.Errorf("timed out waiting for volume %v to become available", volumeId)
}

func assertStateBecomesClosed(c *gc.C, st *state.State) {
	// This is gross but I can't see any other way to check for
	// closedness outside the state package.
	checkModel := func() {
		attempt := utils.AttemptStrategy{
			Total: coretesting.LongWait,
			Delay: coretesting.ShortWait,
		}
		for a := attempt.Start(); a.Next(); {
			// This will panic once the state is closed.
			_, _ = st.Model()
		}
		// If we got here then st is still open.
		st.Close()
	}
	c.Assert(checkModel, gc.PanicMatches, "Session already closed")
}

// It appears that sometimes the lock is not cleared when we expect it to be.
// Capture and log any errors from the Unlock method and retry a few times.
func unlockEnvironmentLock(lock *fslock.Lock) {
	attempts := utils.AttemptStrategy{
		Delay: 50 * time.Millisecond,
		Min:   10,
	}
	var err error
	for a := attempts.Start(); a.Next(); {
		err = lock.Unlock()
		if err == nil {
			return
		}
		if a.HasNext() {
			logger.Debugf("failed to unlock configstore lock: %s, retrying", err)
		}
	}
	logger.Errorf("unable to unlock configstore lock: %s", err)
}

func (s *mongoPingerSuite) TestAgentConnectionsShutDownWhenStateDies(c *gc.C) {
	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	gitjujutesting.MgoServer.Destroy()

	attempt := utils.AttemptStrategy{
		Total: coretesting.LongWait,
		Delay: coretesting.ShortWait,
	}
	for a := attempt.Start(); a.Next(); {
		if err := st.Ping(); err != nil {
			c.Assert(err, gc.ErrorMatches, "connection is shut down")
			return
		}
	}
	c.Fatalf("timed out waiting for API server to die")
}

// GetInstall runs 'apt-get install packages' for the packages listed
// here. apt-get install calls are retried for 30 times with a 10
// second sleep between attempts.
func GetInstall(packages ...string) error {
	cmdArgs := append([]string(nil), getCommand...)
	cmdArgs = append(cmdArgs, "install")
	cmdArgs = append(cmdArgs, packages...)
	logger.Infof("Running: %s", cmdArgs)
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	cmd.Env = append(os.Environ(), getEnvOptions...)

	var err error
	var out []byte
	// Retry APT operations for 30 times, sleeping 10 seconds
	// between attempts. This avoids failure in the case of
	// something else having the dpkg lock (e.g. a charm on the
	// machine we're deploying containers to).
	attempt := utils.AttemptStrategy{Delay: 10 * time.Second, Min: 30}
	for a := attempt.Start(); a.Next(); {
		out, err = CommandOutput(cmd)
		if err == nil {
			return nil
		}
		exitError, ok := err.(*exec.ExitError)
		if !ok {
			err = fmt.Errorf("unexpected error type %T", err)
			break
		}
		waitStatus, ok := exitError.ProcessState.Sys().(syscall.WaitStatus)
		if !ok {
			err = fmt.Errorf("unexpected process state type %T", exitError.ProcessState.Sys())
			break
		}
		// From apt-get(8) "apt-get returns zero on normal
		// operation, decimal 100 on error."
		if waitStatus.ExitStatus() != 100 {
			break
		}
	}
	if err != nil {
		logger.Errorf("apt-get command failed: %v\nargs: %#v\n%s",
			err, cmdArgs, string(out))
		return fmt.Errorf("apt-get failed: %v", err)
	}
	return nil
}

func (v *ebsVolumeSource) waitVolume(
	volumeId string,
	attempt utils.AttemptStrategy,
	pred func(v *ec2.Volume) (bool, error),
) (*ec2.Volume, error) {
	for a := attempt.Start(); a.Next(); {
		volume, err := v.describeVolume(volumeId)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ok, err := pred(volume)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if ok {
			return volume, nil
		}
	}
	return nil, errWaitVolumeTimeout
}