func (c *upgradeWorkerContext) finaliseUpgrade(info *state.UpgradeInfo) error {
	if !c.isStateServer {
		return nil
	}

	if c.isMaster {
		// Tell other state servers that the master has completed its
		// upgrade steps.
		if err := info.SetStatus(state.UpgradeFinishing); err != nil {
			return errors.Annotate(err, "upgrade done but")
		}
	}

	if err := info.SetStateServerDone(c.machineId); err != nil {
		return errors.Annotate(err, "upgrade done but failed to synchronise")
	}

	return nil
}

func (w *upgradesteps) finaliseUpgrade(info *state.UpgradeInfo) error {
	if !w.isController {
		return nil
	}

	if w.isMaster {
		// Tell other controllers that the master has completed its
		// upgrade steps.
		if err := info.SetStatus(state.UpgradeFinishing); err != nil {
			return errors.Annotate(err, "upgrade done but")
		}
	}

	if err := info.SetControllerDone(w.tag.Id()); err != nil {
		return errors.Annotate(err, "upgrade done but failed to synchronise")
	}

	return nil
}

func (s *UpgradeSuite) checkUpgradeInfoArchived(
	c *gc.C,
	initialInfo *state.UpgradeInfo,
	expectedStatus state.UpgradeStatus,
	expectedControllers int,
) {
	info := s.getOneUpgradeInfo(c)
	c.Assert(info.Status(), gc.Equals, expectedStatus)
	c.Assert(info.PreviousVersion(), gc.Equals, initialInfo.PreviousVersion())
	c.Assert(info.TargetVersion(), gc.Equals, initialInfo.TargetVersion())
	// Truncate because mongo only stores times down to millisecond resolution.
	c.Assert(info.Started().Equal(initialInfo.Started().Truncate(time.Millisecond)), jc.IsTrue)
	c.Assert(len(info.ControllersDone()), gc.Equals, expectedControllers)
	if expectedControllers > 0 {
		c.Assert(info.ControllersDone(), jc.SameContents, info.ControllersReady())
	}
}

func (c *upgradeWorkerContext) waitForOtherStateServers(info *state.UpgradeInfo) error {
	watcher := info.Watch()
	defer watcher.Stop()

	maxWait := getUpgradeStartTimeout(c.isMaster)
	timeout := time.After(maxWait)
	for {
		select {
		case <-watcher.Changes():
			if err := info.Refresh(); err != nil {
				return errors.Trace(err)
			}
			if c.isMaster {
				if ready, err := info.AllProvisionedStateServersReady(); err != nil {
					return errors.Trace(err)
				} else if ready {
					// All state servers ready to start upgrade
					err := info.SetStatus(state.UpgradeRunning)
					return errors.Trace(err)
				}
			} else {
				if info.Status() == state.UpgradeFinishing {
					// Master is done, ok to proceed
					return nil
				}
			}
		case <-timeout:
			if c.isMaster {
				if err := info.Abort(); err != nil {
					return errors.Annotate(err, "unable to abort upgrade")
				}
			}
			return errors.Errorf("timed out after %s", maxWait)
		case <-c.agent.Dying():
			return agentTerminating
		}

	}
}

func (s *UpgradeSuite) setToFinishing(c *gc.C, info *state.UpgradeInfo) {
	err := info.SetStatus(state.UpgradeRunning)
	c.Assert(err, jc.ErrorIsNil)
	err = info.SetStatus(state.UpgradeFinishing)
	c.Assert(err, jc.ErrorIsNil)
}

// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades() error {
	from := version.Current
	from.Number = c.agentConfig.UpgradedToVersion()
	if from == version.Current {
		logger.Infof("upgrade to %v already completed.", version.Current)
		return nil
	}

	a := c.agent
	tag, ok := c.agentConfig.Tag().(names.MachineTag)
	if !ok {
		return errors.New("machine agent's tag is not a MachineTag")
	}
	machineId := tag.Id()

	isMaster, err := isMachineMaster(c.st, machineId)
	if err != nil {
		return errors.Trace(err)
	}

	var upgradeInfo *state.UpgradeInfo
	if c.isStateServer {
		upgradeInfo, err = c.st.EnsureUpgradeInfo(machineId, from.Number, version.Current.Number)
		if err != nil {
			return errors.Trace(err)
		}

		// State servers need to wait for other state servers to be
		// ready to run the upgrade.
		logger.Infof("waiting for other state servers to be ready for upgrade")
		if err := c.waitForOtherStateServers(upgradeInfo, isMaster); err != nil {
			if err == agentTerminating {
				logger.Warningf(`stopped waiting for other state servers: %v`, err)
			} else {
				logger.Errorf(`other state servers failed to come up for upgrade `+
					`to %s - aborting: %v`, version.Current, err)
				a.setMachineStatus(c.apiState, params.StatusError,
					fmt.Sprintf("upgrade to %v aborted while waiting for other "+
						"state servers: %v", version.Current, err))
				// If master, trigger a rollback to the previous agent version.
				if isMaster {
					logger.Errorf("downgrading environment agent version to %v due to aborted upgrade",
						from.Number)
					if rollbackErr := c.st.SetEnvironAgentVersion(from.Number); rollbackErr != nil {
						logger.Errorf("rollback failed: %v", rollbackErr)
						return errors.Annotate(rollbackErr, "failed to roll back desired agent version")
					}
				}
			}
			return err
		}
	}

	err = a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
		var upgradeErr error
		a.setMachineStatus(c.apiState, params.StatusStarted,
			fmt.Sprintf("upgrading to %v", version.Current))

		context := upgrades.NewContext(agentConfig, c.apiState, c.st)
		for _, job := range c.jobs {
			target := upgradeTarget(job, isMaster)
			if target == "" {
				continue
			}
			logger.Infof("starting upgrade from %v to %v for %v %q",
				from, version.Current, target, tag)

			attempts := getUpgradeRetryStrategy()
			for attempt := attempts.Start(); attempt.Next(); {
				upgradeErr = upgradesPerformUpgrade(from.Number, target, context)
				if upgradeErr == nil {
					break
				}
				if connectionIsDead(c.apiState) {
					// API connection has gone away - abort!
					return &apiLostDuringUpgrade{upgradeErr}
				}
				retryText := "will retry"
				if !attempt.HasNext() {
					retryText = "giving up"
				}
				logger.Errorf("upgrade from %v to %v for %v %q failed (%s): %v",
					from, version.Current, target, tag, retryText, upgradeErr)
				a.setMachineStatus(c.apiState, params.StatusError,
					fmt.Sprintf("upgrade to %v failed (%s): %v",
						version.Current, retryText, upgradeErr))
			}
		}
		if upgradeErr != nil {
			return upgradeErr
		}
		agentConfig.SetUpgradedToVersion(version.Current.Number)
		return nil
	})
	if err != nil {
		logger.Errorf("upgrade to %v failed: %v", version.Current, err)
		return err
	}
//.........这里部分代码省略.........