// newWebsocketDialer returns a function that
// can be passed to utils/parallel.Try.Start.
func newWebsocketDialer(cfg *websocket.Config, opts DialOpts) func(<-chan struct{}) (io.Closer, error) {
	openAttempt := utils.AttemptStrategy{
		Total: opts.Timeout,
		Delay: opts.RetryDelay,
	}
	return func(stop <-chan struct{}) (io.Closer, error) {
		for a := openAttempt.Start(); a.Next(); {
			select {
			case <-stop:
				return nil, parallel.ErrStopped
			default:
			}
			logger.Infof("dialing %q", cfg.Location)
			conn, err := websocket.DialConfig(cfg)
			if err == nil {
				return conn, nil
			}
			if a.HasNext() {
				logger.Debugf("error dialing %q, will retry: %v", cfg.Location, err)
			} else {
				logger.Infof("error dialing %q: %v", cfg.Location, err)
				return nil, fmt.Errorf("timed out connecting to %q", cfg.Location)
			}
		}
		panic("unreachable")
	}
}

func newServer() (*coretesting.MgoInstance, error) {
	inst := &coretesting.MgoInstance{Params: []string{"--replSet", name}}

	err := inst.Start(true)
	if err != nil {
		return nil, fmt.Errorf("Error starting mongo server: %s", err.Error())
	}

	// by dialing right now, we'll wait until it's running
	strategy := utils.AttemptStrategy{Total: time.Second * 5, Delay: time.Millisecond * 100}
	attempt := strategy.Start()
	for attempt.Next() {
		var session *mgo.Session
		session, err = inst.DialDirect()
		if err != nil {
			err = fmt.Errorf("Error dialing mongo server %q: %s", inst.Addr(), err.Error())
		} else {
			session.SetMode(mgo.Monotonic, true)
			err = session.Ping()
			if err != nil {
				err = fmt.Errorf("Error pinging mongo server %q: %s", inst.Addr(), err.Error())
			}
			session.Close()
		}
		if err == nil || !attempt.HasNext() {
			break
		}
	}
	return inst, err
}

func checkFileHasContents(c *gc.C, stor storage.StorageReader, name string, contents []byte, attempt utils.AttemptStrategy) {
	r, err := storage.GetWithRetry(stor, name, attempt)
	c.Assert(err, gc.IsNil)
	c.Check(r, gc.NotNil)
	defer r.Close()

	data, err := ioutil.ReadAll(r)
	c.Check(err, gc.IsNil)
	c.Check(data, gc.DeepEquals, contents)

	url, err := stor.URL(name)
	c.Assert(err, gc.IsNil)

	var resp *http.Response
	for a := attempt.Start(); a.Next(); {
		resp, err = utils.GetValidatingHTTPClient().Get(url)
		c.Assert(err, gc.IsNil)
		if resp.StatusCode != 404 {
			break
		}
		c.Logf("get retrying after earlier get succeeded. *sigh*.")
	}
	c.Assert(err, gc.IsNil)
	data, err = ioutil.ReadAll(resp.Body)
	c.Assert(err, gc.IsNil)
	defer resp.Body.Close()
	c.Assert(resp.StatusCode, gc.Equals, 200, gc.Commentf("error response: %s", data))
	c.Check(data, gc.DeepEquals, contents)
}

// ListWithRetry lists the files matching prefix from stor using the specified attempt strategy.
func ListWithRetry(stor StorageReader, prefix string, attempt utils.AttemptStrategy) (list []string, err error) {
	for a := attempt.Start(); a.Next(); {
		list, err = stor.List(prefix)
		if err == nil || !stor.ShouldRetry(err) {
			break
		}
	}
	return list, err
}

// GetWithRetry gets the named file from stor using the specified attempt strategy.
func GetWithRetry(stor StorageReader, name string, attempt utils.AttemptStrategy) (r io.ReadCloser, err error) {
	for a := attempt.Start(); a.Next(); {
		r, err = stor.Get(name)
		if err == nil || !stor.ShouldRetry(err) {
			break
		}
	}
	return r, err
}

func (s *mongoPingerSuite) TestAgentConnectionsShutDownWhenStateDies(c *gc.C) {
	s.PatchValue(apiserver.MongoPingInterval, coretesting.ShortWait)
	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, gc.IsNil)
	coretesting.MgoServer.Destroy()

	attempt := utils.AttemptStrategy{
		Total: coretesting.LongWait,
		Delay: coretesting.ShortWait,
	}
	for a := attempt.Start(); a.Next(); {
		if err := st.Ping(); err != nil {
			c.Assert(err, gc.ErrorMatches, "connection is shut down")
			return
		}
	}
	c.Fatalf("timed out waiting for API server to die")
}

func (utilsSuite) TestAttemptTiming(c *gc.C) {
	testAttempt := utils.AttemptStrategy{
		Total: 0.25e9,
		Delay: 0.1e9,
	}
	want := []time.Duration{0, 0.1e9, 0.2e9, 0.2e9}
	got := make([]time.Duration, 0, len(want)) // avoid allocation when testing timing
	t0 := time.Now()
	for a := testAttempt.Start(); a.Next(); {
		got = append(got, time.Now().Sub(t0))
	}
	got = append(got, time.Now().Sub(t0))
	c.Assert(got, gc.HasLen, len(want))
	const margin = 0.01e9
	for i, got := range want {
		lo := want[i] - margin
		hi := want[i] + margin
		if got < lo || got > hi {
			c.Errorf("attempt %d want %g got %g", i, want[i].Seconds(), got.Seconds())
		}
	}
}

func ExampleAttempt_HasNext() {
	// This example shows how Attempt.HasNext can be used to help
	// structure an attempt loop. If the godoc example code allowed
	// us to make the example return an error, we would uncomment
	// the commented return statements.
	attempts := utils.AttemptStrategy{
		Total: 1 * time.Second,
		Delay: 250 * time.Millisecond,
	}
	for attempt := attempts.Start(); attempt.Next(); {
		x, err := doSomething()
		if shouldRetry(err) && attempt.HasNext() {
			continue
		}
		if err != nil {
			// return err
			return
		}
		doSomethingWith(x)
	}
	// return ErrTimedOut
	return
}

func (s *MongoSuite) TestCurrentStatus(c *gc.C) {
	session := root.MustDial()
	defer session.Close()

	inst1, err := newServer()
	c.Assert(err, gc.IsNil)
	defer inst1.Destroy()
	defer Remove(session, inst1.Addr())

	inst2, err := newServer()
	c.Assert(err, gc.IsNil)
	defer inst2.Destroy()
	defer Remove(session, inst2.Addr())

	strategy := utils.AttemptStrategy{Total: time.Second * 31, Delay: time.Millisecond * 100}
	attempt := strategy.Start()
	for attempt.Next() {
		err = Add(session, Member{Address: inst1.Addr()}, Member{Address: inst2.Addr()})
		if err == nil || !attempt.HasNext() {
			break
		}
	}
	c.Assert(err, gc.IsNil)

	expected := &Status{
		Name: name,
		Members: []MemberStatus{{
			Id:      1,
			Address: root.Addr(),
			Self:    true,
			ErrMsg:  "",
			Healthy: true,
			State:   PrimaryState,
		}, {
			Id:      2,
			Address: inst1.Addr(),
			Self:    false,
			ErrMsg:  "",
			Healthy: true,
			State:   SecondaryState,
		}, {
			Id:      3,
			Address: inst2.Addr(),
			Self:    false,
			ErrMsg:  "",
			Healthy: true,
			State:   SecondaryState,
		}},
	}

	strategy.Total = time.Second * 90
	attempt = strategy.Start()
	var res *Status
	for attempt.Next() {
		var err error
		res, err = CurrentStatus(session)
		if err != nil {
			if !attempt.HasNext() {
				c.Errorf("Couldn't get status before timeout, got err: %v", err)
				return
			} else {
				// try again
				continue
			}
		}

		if res.Members[0].State == PrimaryState &&
			res.Members[1].State == SecondaryState &&
			res.Members[2].State == SecondaryState {
			break
		}
		if !attempt.HasNext() {
			c.Errorf("Servers did not get into final state before timeout.  Status: %#v", res)
			return
		}
	}

	for x, _ := range res.Members {
		// non-empty uptime and ping
		c.Check(res.Members[x].Uptime, gc.Not(gc.Equals), 0)

		// ping is always going to be zero since we're on localhost
		// so we can't really test it right now

		// now overwrite Uptime so it won't throw off DeepEquals
		res.Members[x].Uptime = 0
	}
	c.Check(res, jc.DeepEquals, expected)
}

func (s *MongoSuite) TestAddRemoveSet(c *gc.C) {
	session := root.MustDial()
	defer session.Close()

	members := make([]Member, 0, 5)

	// Add should be idempotent, so re-adding root here shouldn't result in
	// two copies of root in the replica set
	members = append(members, Member{Address: root.Addr(), Tags: initialTags})

	instances := make([]*coretesting.MgoInstance, 0, 5)
	instances = append(instances, root)

	for x := 0; x < 4; x++ {
		inst, err := newServer()
		c.Assert(err, gc.IsNil)
		instances = append(instances, inst)
		defer inst.Destroy()
		defer Remove(session, inst.Addr())

		key := fmt.Sprintf("key%d", x)
		val := fmt.Sprintf("val%d", x)

		tags := map[string]string{key: val}

		members = append(members, Member{Address: inst.Addr(), Tags: tags})
	}

	var err error

	// We use a delay of 31s. Our Mongo Dial timeout is 15s, so this gives
	// us 2 attempts before we give up.
	strategy := utils.AttemptStrategy{Total: time.Second * 31, Delay: time.Millisecond * 100}
	start := time.Now()
	attemptCount := 0
	attempt := strategy.Start()
	for attempt.Next() {
		attemptCount += 1
		err = Add(session, members...)
		if err == nil || !attempt.HasNext() {
			break
		}
		c.Logf("attempting to Add got error: %v", err)
	}
	c.Logf("Add() %d attempts in %s", attemptCount, time.Since(start))
	c.Assert(err, gc.IsNil)

	expectedMembers := make([]Member, len(members))
	for x, m := range members {
		// Ids should start at 1 (for the root) and go up
		m.Id = x + 1
		expectedMembers[x] = m
	}

	var cfg *Config
	start = time.Now()
	attemptCount = 0
	attempt = strategy.Start()
	for attempt.Next() {
		attemptCount += 1
		cfg, err = CurrentConfig(session)
		if err == nil || !attempt.HasNext() {
			break
		}
		c.Logf("attempting CurrentConfig got error: %v", err)
	}
	c.Logf("CurrentConfig() %d attempts in %s", attemptCount, time.Since(start))
	c.Assert(err, gc.IsNil)
	c.Assert(cfg.Name, gc.Equals, name)

	// 2 since we already changed it once
	c.Assert(cfg.Version, gc.Equals, 2)

	mems := cfg.Members

	c.Assert(mems, jc.DeepEquals, expectedMembers)

	// Now remove the last two Members
	start = time.Now()
	attemptCount = 0
	attempt = strategy.Start()
	for attempt.Next() {
		attemptCount += 1
		err = Remove(session, members[3].Address, members[4].Address)
		if err == nil || !attempt.HasNext() {
			break
		}
		c.Logf("attempting Remove got error: %v", err)
	}
	c.Logf("Remove() %d attempts in %s", attemptCount, time.Since(start))
	c.Assert(err, gc.IsNil)

	expectedMembers = expectedMembers[0:3]

	start = time.Now()
	attemptCount = 0
	attempt = strategy.Start()
	for attempt.Next() {
		attemptCount += 1
		mems, err = CurrentMembers(session)
//.........这里部分代码省略.........

func (utilsSuite) TestAttemptNextHasNext(c *gc.C) {
	a := utils.AttemptStrategy{}.Start()
	c.Assert(a.Next(), gc.Equals, true)
	c.Assert(a.Next(), gc.Equals, false)

	a = utils.AttemptStrategy{}.Start()
	c.Assert(a.Next(), gc.Equals, true)
	c.Assert(a.HasNext(), gc.Equals, false)
	c.Assert(a.Next(), gc.Equals, false)

	a = utils.AttemptStrategy{Total: 2e8}.Start()
	c.Assert(a.Next(), gc.Equals, true)
	c.Assert(a.HasNext(), gc.Equals, true)
	time.Sleep(2e8)
	c.Assert(a.HasNext(), gc.Equals, true)
	c.Assert(a.Next(), gc.Equals, true)
	c.Assert(a.Next(), gc.Equals, false)

	a = utils.AttemptStrategy{Total: 1e8, Min: 2}.Start()
	time.Sleep(1e8)
	c.Assert(a.Next(), gc.Equals, true)
	c.Assert(a.HasNext(), gc.Equals, true)
	c.Assert(a.Next(), gc.Equals, true)
	c.Assert(a.HasNext(), gc.Equals, false)
	c.Assert(a.Next(), gc.Equals, false)
}

// startReplicaSet starts up a replica set with n mongo instances.
func startReplicaSet(n int) (_ []*testing.MgoInstance, err error) {
	insts := make([]*testing.MgoInstance, 0, n)
	root, err := newMongoInstance()
	if err != nil {
		return nil, err
	}
	insts = append(insts, root)
	defer func() {
		if err == nil {
			return
		}
		for _, inst := range insts {
			inst.Destroy()
		}
	}()

	dialInfo := root.DialInfo()
	dialInfo.Direct = true
	dialInfo.Timeout = 60 * time.Second

	session, err := root.DialDirect()
	if err != nil {
		return nil, fmt.Errorf("cannot dial root instance: %v", err)
	}
	defer session.Close()

	logger.Infof("dialled root instance")

	if err := replicaset.Initiate(session, root.Addr(), replicaSetName, nil); err != nil {
		return nil, fmt.Errorf("cannot initiate replica set: %v", err)
	}
	var members []replicaset.Member
	for i := 1; i < n; i++ {
		inst, err := newMongoInstance()
		if err != nil {
			return nil, err
		}
		insts = append(insts, inst)
		members = append(members, replicaset.Member{
			Address:  inst.Addr(),
			Priority: newFloat64(0.1),
			Id:       i + 1,
		})
	}
	attempt := utils.AttemptStrategy{
		Total: 60 * time.Second,
		Delay: 1 * time.Second,
	}
	for a := attempt.Start(); a.Next(); {
		err := replicaset.Add(session, members...)
		if err == nil {
			break
		}
		logger.Errorf("cannot add members: %v", err)
		if !a.HasNext() {
			return nil, fmt.Errorf("timed out trying to add members")
		}
		logger.Errorf("retrying")
	}
	return insts, err
}