func (*functionSuite) TestMaskf(c *gc.C) {
	first := errors.New("first")
	err := errors.Maskf(first, "masked %d", 42) //err maskfTest
	c.Assert(err.Error(), gc.Equals, "masked 42: first")
	c.Assert(errors.Cause(err), gc.Equals, err)
	c.Assert(errors.Details(err), jc.Contains, tagToLocation["maskfTest"].String())

	c.Assert(errors.Maskf(nil, "mask"), gc.IsNil)
}

// Write atomically writes to disk the relation state change in hi.
// It must be called after the respective hook was executed successfully.
// Write doesn't validate hi but guarantees that successive writes of
// the same hi are idempotent.
func (d *StateDir) Write(hi hook.Info) (err error) {
	defer errors.Maskf(&err, "failed to write %q hook info for %q on state directory", hi.Kind, hi.RemoteUnit)
	if hi.Kind == hooks.RelationBroken {
		return d.Remove()
	}
	name := strings.Replace(hi.RemoteUnit, "/", "-", 1)
	path := filepath.Join(d.path, name)
	if hi.Kind == hooks.RelationDeparted {
		if err = os.Remove(path); err != nil && !os.IsNotExist(err) {
			return err
		}
		// If atomic delete succeeded, update own state.
		delete(d.state.Members, hi.RemoteUnit)
		return nil
	}
	di := diskInfo{&hi.ChangeVersion, hi.Kind == hooks.RelationJoined}
	if err := utils.WriteYaml(path, &di); err != nil {
		return err
	}
	// If write was successful, update own state.
	d.state.Members[hi.RemoteUnit] = hi.ChangeVersion
	if hi.Kind == hooks.RelationJoined {
		d.state.ChangedPending = hi.RemoteUnit
	} else {
		d.state.ChangedPending = ""
	}
	return nil
}

// Validate returns an error if the supplied hook.Info does not represent
// a valid change to the relation state. Hooks must always be validated
// against the current state before they are run, to ensure that the system
// meets its guarantees about hook execution order.
func (s *State) Validate(hi hook.Info) (err error) {
	defer errors.Maskf(&err, "inappropriate %q for %q", hi.Kind, hi.RemoteUnit)
	if hi.RelationId != s.RelationId {
		return fmt.Errorf("expected relation %d, got relation %d", s.RelationId, hi.RelationId)
	}
	if s.Members == nil {
		return fmt.Errorf(`relation is broken and cannot be changed further`)
	}
	unit, kind := hi.RemoteUnit, hi.Kind
	if kind == hooks.RelationBroken {
		if len(s.Members) == 0 {
			return nil
		}
		return fmt.Errorf(`cannot run "relation-broken" while units still present`)
	}
	if s.ChangedPending != "" {
		if unit != s.ChangedPending || kind != hooks.RelationChanged {
			return fmt.Errorf(`expected "relation-changed" for %q`, s.ChangedPending)
		}
	} else if _, joined := s.Members[unit]; joined && kind == hooks.RelationJoined {
		return fmt.Errorf("unit already joined")
	} else if !joined && kind != hooks.RelationJoined {
		return fmt.Errorf("unit has not joined")
	}
	return nil
}

func (*functionSuite) TestCause(c *gc.C) {
	c.Assert(errors.Cause(nil), gc.IsNil)
	c.Assert(errors.Cause(someErr), gc.Equals, someErr)

	fmtErr := fmt.Errorf("simple")
	c.Assert(errors.Cause(fmtErr), gc.Equals, fmtErr)

	err := errors.Wrap(someErr, fmtErr)
	c.Assert(errors.Cause(err), gc.Equals, fmtErr)

	err = errors.Annotate(err, "annotated")
	c.Assert(errors.Cause(err), gc.Equals, fmtErr)

	err = errors.Maskf(err, "maksed")
	c.Assert(errors.Cause(err), gc.Equals, err)

	// Look for a file that we know isn't there.
	dir := c.MkDir()
	_, err = os.Stat(filepath.Join(dir, "not-there"))
	c.Assert(os.IsNotExist(err), jc.IsTrue)

	err = errors.Annotatef(err, "wrap it")
	// Now the error itself isn't a 'IsNotExist'.
	c.Assert(os.IsNotExist(err), jc.IsFalse)
	// However if we use the Check method, it is.
	c.Assert(os.IsNotExist(errors.Cause(err)), jc.IsTrue)
}

// Destroy ensures that the service and all its relations will be removed at
// some point; if the service has no units, and no relation involving the
// service has any units in scope, they are all removed immediately.
func (s *Service) Destroy() (err error) {
	defer errors.Maskf(&err, "cannot destroy service %q", s)
	defer func() {
		if err == nil {
			// This is a white lie; the document might actually be removed.
			s.doc.Life = Dying
		}
	}()
	svc := &Service{st: s.st, doc: s.doc}
	for i := 0; i < 5; i++ {
		switch ops, err := svc.destroyOps(); err {
		case errRefresh:
		case errAlreadyDying:
			return nil
		case nil:
			if err := svc.st.runTransaction(ops); err != txn.ErrAborted {
				return err
			}
		default:
			return err
		}
		if err := svc.Refresh(); errors.IsNotFound(err) {
			return nil
		} else if err != nil {
			return err
		}
	}
	return ErrExcessiveContention
}

// Destroy ensures that the service and all its relations will be removed at
// some point; if the service has no units, and no relation involving the
// service has any units in scope, they are all removed immediately.
func (s *Service) Destroy() (err error) {
	defer errors.Maskf(&err, "cannot destroy service %q", s)
	defer func() {
		if err == nil {
			// This is a white lie; the document might actually be removed.
			s.doc.Life = Dying
		}
	}()
	svc := &Service{st: s.st, doc: s.doc}
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			if err := svc.Refresh(); errors.IsNotFound(err) {
				return nil, jujutxn.ErrNoOperations
			} else if err != nil {
				return nil, err
			}
		}
		switch ops, err := svc.destroyOps(); err {
		case errRefresh:
		case errAlreadyDying:
			return nil, jujutxn.ErrNoOperations
		case nil:
			return ops, nil
		default:
			return nil, err
		}
		return nil, jujutxn.ErrTransientFailure
	}
	return s.st.run(buildTxn)
}

// AssignUnit places the unit on a machine. Depending on the policy, and the
// state of the environment, this may lead to new instances being launched
// within the environment.
func (st *State) AssignUnit(u *Unit, policy AssignmentPolicy) (err error) {
	if !u.IsPrincipal() {
		return fmt.Errorf("subordinate unit %q cannot be assigned directly to a machine", u)
	}
	defer errors.Maskf(&err, "cannot assign unit %q to machine", u)
	var m *Machine
	switch policy {
	case AssignLocal:
		m, err = st.Machine("0")
		if err != nil {
			return err
		}
		return u.AssignToMachine(m)
	case AssignClean:
		if _, err = u.AssignToCleanMachine(); err != noCleanMachines {
			return err
		}
		return u.AssignToNewMachineOrContainer()
	case AssignCleanEmpty:
		if _, err = u.AssignToCleanEmptyMachine(); err != noCleanMachines {
			return err
		}
		return u.AssignToNewMachineOrContainer()
	case AssignNew:
		return u.AssignToNewMachine()
	}
	return fmt.Errorf("unknown unit assignment policy: %q", policy)
}

// SetConstraints replaces the current service constraints.
func (s *Service) SetConstraints(cons constraints.Value) (err error) {
	unsupported, err := s.st.validateConstraints(cons)
	if len(unsupported) > 0 {
		logger.Warningf(
			"setting constraints on service %q: unsupported constraints: %v", s.Name(), strings.Join(unsupported, ","))
	} else if err != nil {
		return err
	}
	if s.doc.Subordinate {
		return ErrSubordinateConstraints
	}
	defer errors.Maskf(&err, "cannot set constraints")
	if s.doc.Life != Alive {
		return errNotAlive
	}
	ops := []txn.Op{
		{
			C:      s.st.services.Name,
			Id:     s.doc.Name,
			Assert: isAliveDoc,
		},
		setConstraintsOp(s.st, s.globalKey(), cons),
	}
	return onAbort(s.st.runTransaction(ops), errNotAlive)
}

// UpdateCloudCredential adds or updates a cloud credential with the given tag.
func (st *State) UpdateCloudCredential(tag names.CloudCredentialTag, credential cloud.Credential) error {
	credentials := map[names.CloudCredentialTag]cloud.Credential{tag: credential}
	buildTxn := func(attempt int) ([]txn.Op, error) {
		cloudName := tag.Cloud().Id()
		cloud, err := st.Cloud(cloudName)
		if err != nil {
			return nil, errors.Trace(err)
		}
		ops, err := validateCloudCredentials(cloud, cloudName, credentials)
		if err != nil {
			return nil, errors.Annotate(err, "validating cloud credentials")
		}
		_, err = st.CloudCredential(tag)
		if err != nil && !errors.IsNotFound(err) {
			return nil, errors.Maskf(err, "fetching cloud credentials")
		}
		if err == nil {
			ops = append(ops, updateCloudCredentialOp(tag, credential))
		} else {
			ops = append(ops, createCloudCredentialOp(tag, credential))
		}
		return ops, nil
	}
	if err := st.run(buildTxn); err != nil {
		return errors.Annotate(err, "updating cloud credentials")
	}
	return nil
}

// validate returns an error if the state violates expectations.
func (st State) validate() (err error) {
	defer errors.Maskf(&err, "invalid uniter state")
	hasHook := st.Hook != nil
	hasCharm := st.CharmURL != nil
	switch st.Op {
	case Install:
		if hasHook {
			return fmt.Errorf("unexpected hook info")
		}
		fallthrough
	case Upgrade:
		if !hasCharm {
			return fmt.Errorf("missing charm URL")
		}
	case Continue, RunHook:
		if !hasHook {
			return fmt.Errorf("missing hook info")
		} else if hasCharm {
			return fmt.Errorf("unexpected charm URL")
		}
	default:
		return fmt.Errorf("unknown operation %q", st.Op)
	}
	switch st.OpStep {
	case Queued, Pending, Done:
	default:
		return fmt.Errorf("unknown operation step %q", st.OpStep)
	}
	if hasHook {
		return st.Hook.Validate()
	}
	return nil
}

// Destroy ensures that the relation will be removed at some point; if no units
// are currently in scope, it will be removed immediately.
func (r *Relation) Destroy() (err error) {
	defer errors.Maskf(&err, "cannot destroy relation %q", r)
	if len(r.doc.Endpoints) == 1 && r.doc.Endpoints[0].Role == charm.RolePeer {
		return fmt.Errorf("is a peer relation")
	}
	defer func() {
		if err == nil {
			// This is a white lie; the document might actually be removed.
			r.doc.Life = Dying
		}
	}()
	rel := &Relation{r.st, r.doc}
	// In this context, aborted transactions indicate that the number of units
	// in scope have changed between 0 and not-0. The chances of 5 successive
	// attempts each hitting this change -- which is itself an unlikely one --
	// are considered to be extremely small.
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			if err := rel.Refresh(); errors.IsNotFound(err) {
				return []txn.Op{}, nil
			} else if err != nil {
				return nil, err
			}
		}
		ops, _, err := rel.destroyOps("")
		if err == errAlreadyDying {
			return nil, jujutxn.ErrNoOperations
		} else if err != nil {
			return nil, err
		}
		return ops, nil
	}
	return rel.st.run(buildTxn)
}

// ReadAllStateDirs loads and returns every StateDir persisted directly inside
// the supplied dirPath. If dirPath does not exist, no error is returned.
func ReadAllStateDirs(dirPath string) (dirs map[int]*StateDir, err error) {
	defer errors.Maskf(&err, "cannot load relations state from %q", dirPath)
	if _, err := os.Stat(dirPath); os.IsNotExist(err) {
		return nil, nil
	} else if err != nil {
		return nil, err
	}
	fis, err := ioutil.ReadDir(dirPath)
	if err != nil {
		return nil, err
	}
	dirs = map[int]*StateDir{}
	for _, fi := range fis {
		// Entries with integer names must be directories containing StateDir
		// data; all other names will be ignored.
		relationId, err := strconv.Atoi(fi.Name())
		if err != nil {
			// This doesn't look like a relation.
			continue
		}
		dir, err := ReadStateDir(dirPath, relationId)
		if err != nil {
			return nil, err
		}
		dirs[relationId] = dir
	}
	return dirs, nil
}

// vpcNotUsablef returns an error which satisfies isVPCNotUsableError().
func vpcNotUsablef(optionalCause error, format string, args ...interface{}) error {
	outerErr := errors.Errorf(format, args...)
	if optionalCause != nil {
		outerErr = errors.Maskf(optionalCause, format, args...)
	}

	innerErr, _ := outerErr.(*errors.Err) // cannot fail.
	return &vpcNotUsableError{*innerErr}
}

// FinishMachineConfig sets fields on a MachineConfig that can be determined by
// inspecting a plain config.Config and the machine constraints at the last
// moment before bootstrapping. It assumes that the supplied Config comes from
// an environment that has passed through all the validation checks in the
// Bootstrap func, and that has set an agent-version (via finding the tools to,
// use for bootstrap, or otherwise).
// TODO(fwereade) This function is not meant to be "good" in any serious way:
// it is better that this functionality be collected in one place here than
// that it be spread out across 3 or 4 providers, but this is its only
// redeeming feature.
func FinishMachineConfig(mcfg *cloudinit.MachineConfig, cfg *config.Config, cons constraints.Value) (err error) {
	defer errors.Maskf(&err, "cannot complete machine configuration")

	if err := PopulateMachineConfig(
		mcfg,
		cfg.Type(),
		cfg.AuthorizedKeys(),
		cfg.SSLHostnameVerification(),
		cfg.ProxySettings(),
		cfg.AptProxySettings(),
		cfg.PreferIPv6(),
	); err != nil {
		return err
	}

	// The following settings are only appropriate at bootstrap time. At the
	// moment, the only state server is the bootstrap node, but this
	// will probably change.
	if !mcfg.Bootstrap {
		return nil
	}
	if mcfg.APIInfo != nil || mcfg.MongoInfo != nil {
		return fmt.Errorf("machine configuration already has api/state info")
	}
	caCert, hasCACert := cfg.CACert()
	if !hasCACert {
		return fmt.Errorf("environment configuration has no ca-cert")
	}
	password := cfg.AdminSecret()
	if password == "" {
		return fmt.Errorf("environment configuration has no admin-secret")
	}
	passwordHash := utils.UserPasswordHash(password, utils.CompatSalt)
	mcfg.APIInfo = &api.Info{Password: passwordHash, CACert: caCert}
	mcfg.MongoInfo = &authentication.MongoInfo{Password: passwordHash, Info: mongo.Info{CACert: caCert}}

	// These really are directly relevant to running a state server.
	cert, key, err := cfg.GenerateStateServerCertAndKey()
	if err != nil {
		return errors.Annotate(err, "cannot generate state server certificate")
	}

	srvInfo := params.StateServingInfo{
		StatePort:      cfg.StatePort(),
		APIPort:        cfg.APIPort(),
		Cert:           string(cert),
		PrivateKey:     string(key),
		SystemIdentity: mcfg.SystemPrivateSSHKey,
	}
	mcfg.StateServingInfo = &srvInfo
	mcfg.Constraints = cons
	if mcfg.Config, err = BootstrapConfig(cfg); err != nil {
		return err
	}

	return nil
}

// Remove removes the network interface from state.
func (ni *NetworkInterface) Remove() (err error) {
	defer errors.Maskf(&err, "cannot remove network interface %q", ni)

	ops := []txn.Op{{
		C:      networkInterfacesC,
		Id:     ni.doc.Id,
		Remove: true,
	}}
	// The only abort conditions in play indicate that the network interface
	// has already been removed.
	return onAbort(ni.st.runTransaction(ops), nil)
}

// SetProvisioned sets the provider specific machine id, nonce and also metadata for
// this machine. Once set, the instance id cannot be changed.
//
// When provisioning an instance, a nonce should be created and passed
// when starting it, before adding the machine to the state. This means
// that if the provisioner crashes (or its connection to the state is
// lost) after starting the instance, we can be sure that only a single
// instance will be able to act for that machine.
func (m *Machine) SetProvisioned(id instance.Id, nonce string, characteristics *instance.HardwareCharacteristics) (err error) {
	defer errors.Maskf(&err, "cannot set instance data for machine %q", m)

	if id == "" || nonce == "" {
		return fmt.Errorf("instance id and nonce cannot be empty")
	}

	if characteristics == nil {
		characteristics = &instance.HardwareCharacteristics{}
	}
	instData := &instanceData{
		Id:         m.doc.Id,
		InstanceId: id,
		Arch:       characteristics.Arch,
		Mem:        characteristics.Mem,
		RootDisk:   characteristics.RootDisk,
		CpuCores:   characteristics.CpuCores,
		CpuPower:   characteristics.CpuPower,
		Tags:       characteristics.Tags,
	}
	// SCHEMACHANGE
	// TODO(wallyworld) - do not check instanceId on machineDoc after schema is upgraded
	notSetYet := bson.D{{"instanceid", ""}, {"nonce", ""}}
	ops := []txn.Op{
		{
			C:      m.st.machines.Name,
			Id:     m.doc.Id,
			Assert: append(isAliveDoc, notSetYet...),
			Update: bson.D{{"$set", bson.D{{"instanceid", id}, {"nonce", nonce}}}},
		}, {
			C:      m.st.instanceData.Name,
			Id:     m.doc.Id,
			Assert: txn.DocMissing,
			Insert: instData,
		},
	}

	if err = m.st.runTransaction(ops); err == nil {
		m.doc.Nonce = nonce
		// SCHEMACHANGE
		// TODO(wallyworld) - remove this backward compatibility code when schema upgrades are possible
		// (InstanceId is stored on the instanceData document but we duplicate the value on the machineDoc.
		m.doc.InstanceId = id
		return nil
	} else if err != txn.ErrAborted {
		return err
	} else if alive, err := isAlive(m.st.machines, m.doc.Id); err != nil {
		return err
	} else if !alive {
		return errNotAlive
	}
	return fmt.Errorf("already set")
}

func serviceRelations(st *State, name string) (relations []*Relation, err error) {
	defer errors.Maskf(&err, "can't get relations for service %q", name)
	docs := []relationDoc{}
	err = st.relations.Find(bson.D{{"endpoints.servicename", name}}).All(&docs)
	if err != nil {
		return nil, err
	}
	for _, v := range docs {
		relations = append(relations, newRelation(st, &v))
	}
	return relations, nil
}

// download fetches the supplied charm and checks that it has the correct sha256
// hash, then copies it into the directory. If a value is received on abort, the
// download will be stopped.
func (d *BundlesDir) download(info BundleInfo, abort <-chan struct{}) (err error) {
	archiveURL, disableSSLHostnameVerification, err := info.ArchiveURL()
	if err != nil {
		return err
	}
	defer errors.Maskf(&err, "failed to download charm %q from %q", info.URL(), archiveURL)
	dir := d.downloadsPath()
	if err := os.MkdirAll(dir, 0755); err != nil {
		return err
	}
	aurl := archiveURL.String()
	logger.Infof("downloading %s from %s", info.URL(), aurl)
	if disableSSLHostnameVerification {
		logger.Infof("SSL hostname verification disabled")
	}
	dl := downloader.New(aurl, dir, disableSSLHostnameVerification)
	defer dl.Stop()
	for {
		select {
		case <-abort:
			logger.Infof("download aborted")
			return fmt.Errorf("aborted")
		case st := <-dl.Done():
			if st.Err != nil {
				return st.Err
			}
			logger.Infof("download complete")
			defer st.File.Close()
			actualSha256, _, err := utils.ReadSHA256(st.File)
			if err != nil {
				return err
			}
			archiveSha256, err := info.ArchiveSha256()
			if err != nil {
				return err
			}
			if actualSha256 != archiveSha256 {
				return fmt.Errorf(
					"expected sha256 %q, got %q", archiveSha256, actualSha256,
				)
			}
			logger.Infof("download verified")
			if err := os.MkdirAll(d.path, 0755); err != nil {
				return err
			}
			// Renaming an open file is not possible on Windows
			st.File.Close()
			return os.Rename(st.File.Name(), d.bundlePath(info))
		}
	}
}

// SetAnnotations adds key/value pairs to annotations in MongoDB.
func (a *annotator) SetAnnotations(pairs map[string]string) (err error) {
	defer errors.Maskf(&err, "cannot update annotations on %s", a.tag)
	if len(pairs) == 0 {
		return nil
	}
	// Collect in separate maps pairs to be inserted/updated or removed.
	toRemove := make(map[string]bool)
	toInsert := make(map[string]string)
	toUpdate := make(map[string]string)
	for key, value := range pairs {
		if strings.Contains(key, ".") {
			return fmt.Errorf("invalid key %q", key)
		}
		if value == "" {
			toRemove["annotations."+key] = true
		} else {
			toInsert[key] = value
			toUpdate["annotations."+key] = value
		}
	}
	// Two attempts should be enough to update annotations even with racing
	// clients - if the document does not already exist, one of the clients
	// will create it and the others will fail, then all the rest of the
	// clients should succeed on their second attempt. If the referred-to
	// entity has disappeared, and removed its annotations in the meantime,
	// we consider that worthy of an error (will be fixed when new entities
	// can never share names with old ones).
	for i := 0; i < 2; i++ {
		var ops []txn.Op
		if count, err := a.st.annotations.FindId(a.globalKey).Count(); err != nil {
			return err
		} else if count == 0 {
			// Check that the annotator entity was not previously destroyed.
			if i != 0 {
				return fmt.Errorf("%s no longer exists", a.tag)
			}
			ops, err = a.insertOps(toInsert)
			if err != nil {
				return err
			}
		} else {
			ops = a.updateOps(toUpdate, toRemove)
		}
		if err := a.st.runTransaction(ops); err == nil {
			return nil
		} else if err != txn.ErrAborted {
			return err
		}
	}
	return ErrExcessiveContention
}

// ReadSettings returns a map holding the settings of the unit with the
// supplied name within this relation. An error will be returned if the
// relation no longer exists, or if the unit's service is not part of the
// relation, or the settings are invalid; but mere non-existence of the
// unit is not grounds for an error, because the unit settings are
// guaranteed to persist for the lifetime of the relation, regardless
// of the lifetime of the unit.
func (ru *RelationUnit) ReadSettings(uname string) (m map[string]interface{}, err error) {
	defer errors.Maskf(&err, "cannot read settings for unit %q in relation %q", uname, ru.relation)
	if !names.IsValidUnit(uname) {
		return nil, fmt.Errorf("%q is not a valid unit name", uname)
	}
	key, err := ru.key(uname)
	if err != nil {
		return nil, err
	}
	node, err := readSettings(ru.st, key)
	if err != nil {
		return nil, err
	}
	return node.Map(), nil
}