func StartServicesAndChains(do *definitions.Do) error {
	// start the services and chains
	doSrvs := definitions.NowDo()
	if do.Action.Dependencies == nil || len(do.Action.Dependencies.Services) == 0 {
		log.Debug("No services to start")
	} else {
		doSrvs.Operations.Args = do.Action.Dependencies.Services
		log.WithField("args", doSrvs.Operations.Args).Debug("Starting services")
		if err := services.StartService(doSrvs); err != nil {
			return err
		}
	}

	doChns := definitions.NowDo()
	doChns.Name = do.Action.Chain
	if doChns.Name == "" {
		log.Debug("No chain to start")
	} else {
		log.WithField("=>", doChns.Name).Debug("Starting chain")
		if err := chains.StartChain(do); err != nil {
			return err
		}
	}

	return nil
}

// Add a new entry (name) to the top of the HEAD file
// Expects the chain type and head (id) to be full (already resolved)
func ChangeHead(name string) error {
	if !IsKnownChain(name) && name != "" {
		log.Debug("Chain name not known. Not saving")
		return nil
	}

	log.Debug("Chain name known (or blank). Saving to head file")
	// read in the entire head file and clip
	// if we have reached the max length
	b, err := ioutil.ReadFile(common.HEAD)
	if err != nil {
		return err
	}
	bspl := strings.Split(string(b), "\n")
	var bsp string
	if len(bspl) >= MaxHead {
		bsp = strings.Join(bspl[:MaxHead-1], "\n")
	} else {
		bsp = string(b)
	}

	// add the new head
	var s string
	// handle empty head
	s = name + "\n" + bsp
	err = ioutil.WriteFile(common.HEAD, []byte(s), 0666)
	if err != nil {
		return err
	}

	log.Debug("Head file saved")
	return nil
}

//------- helpers --------
func checkIPFSnotRunning() {
	//os.Setenv("ERIS_IPFS_HOST", "http://0.0.0.0") //conflicts with docker-machine
	do := def.NowDo()
	do.Known = false
	do.Existing = false
	do.Running = true
	do.Quiet = true
	log.Debug("Finding the running services")
	if err := list.ListAll(do, "services"); err != nil {
		IfExit(err)
	}
	res := strings.Split(do.Result, "\n")
	for _, r := range res {
		if r == "ipfs" {
			IfExit(fmt.Errorf("IPFS service is running.\nPlease stop it with.\neris services stop -rx ipfs\n"))
		}
	}
	// make sure ipfs container does not exist
	do = def.NowDo()
	do.Known = false
	do.Existing = true
	do.Running = false
	do.Quiet = true
	log.Debug("Finding the existing services")
	if err := list.ListAll(do, "services"); err != nil {
		IfExit(err)
	}
	res = strings.Split(do.Result, "\n")
	for _, r := range res {
		if r == "ipfs" {
			IfExit(fmt.Errorf("IPFS service exists.\nPlease remove it with\neris services rm ipfs\n"))
		}
	}
}

// ----------------------------------------------------------------------------
// ---------------------    Container Core ------------------------------------
// ----------------------------------------------------------------------------
func createContainer(opts docker.CreateContainerOptions) (*docker.Container, error) {
	dockerContainer, err := util.DockerClient.CreateContainer(opts)
	if err != nil {
		if err == docker.ErrNoSuchImage {
			if os.Getenv("ERIS_PULL_APPROVE") != "true" {
				var input string
				log.WithField("image", opts.Config.Image).Warn("The docker image not found locally")
				fmt.Print("Would you like the marmots to pull it from the repository? (y/n): ")
				fmt.Scanln(&input)

				if input == "Y" || input == "y" || input == "YES" || input == "Yes" || input == "yes" {
					log.Debug("User assented to pull")
				} else {
					log.Debug("User refused to pull")
					return nil, fmt.Errorf("Cannot start a container based on an image you will not let me pull.\n")
				}
			} else {
				log.WithField("image", opts.Config.Image).Warn("The Docker image is not found locally")
				log.Warn("The marmots are approved to pull it from the repository on your behalf")
				log.Warn("This could take a few minutes")
			}
			if err := pullImage(opts.Config.Image, nil); err != nil {
				return nil, err
			}
			dockerContainer, err = util.DockerClient.CreateContainer(opts)
			if err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}
	return dockerContainer, nil
}

func StartService(do *definitions.Do) (err error) {
	var services []*definitions.ServiceDefinition

	do.Operations.Args = append(do.Operations.Args, do.ServicesSlice...)
	log.WithField("args", do.Operations.Args).Info("Building services group")
	for _, srv := range do.Operations.Args {
		s, e := BuildServicesGroup(srv, do.Operations.ContainerNumber)
		if e != nil {
			return e
		}
		services = append(services, s...)
	}

	// [csk]: controls for ops reconciliation, overwrite will, e.g., merge the maps and stuff
	for _, s := range services {
		util.Merge(s.Operations, do.Operations)
	}

	log.Debug("Preparing to build chain")
	for _, s := range services {
		log.WithFields(log.Fields{
			"name":         s.Name,
			"dependencies": s.Dependencies,
			"links":        s.Service.Links,
			"volumes from": s.Service.VolumesFrom,
		}).Debug()

		// Spacer.
		log.Debug()
	}
	services, err = BuildChainGroup(do.ChainName, services)
	if err != nil {
		return err
	}
	log.Debug("Checking services after build chain")
	for _, s := range services {
		log.WithFields(log.Fields{
			"name":         s.Name,
			"dependencies": s.Dependencies,
			"links":        s.Service.Links,
			"volumes from": s.Service.VolumesFrom,
		}).Debug()

		// Spacer.
		log.Debug()
	}

	// NOTE: the top level service should be at the end of the list
	topService := services[len(services)-1]
	topService.Service.Environment = append(topService.Service.Environment, do.Env...)
	topService.Service.Links = append(topService.Service.Links, do.Links...)
	services[len(services)-1] = topService

	return StartGroup(services)
}

func getMachineDeets(machName string) (string, string, error) {
	var out = new(bytes.Buffer)
	var out2 = new(bytes.Buffer)

	noConnectError := fmt.Errorf("Could not evaluate the env vars for the %s docker-machine.\n", machName)
	dHost, dPath := popHostAndPath()

	if (dHost != "" && dPath != "") && (machName == "eris" || machName == "default") {
		return dHost, dPath, nil
	}

	// TODO: when go-dockerclient adds machine API endpoints use those instead.
	log.WithField("machine", machName).Debug("Querying Docker Machine URL")
	cmd := exec.Command("docker-machine", "url", machName)
	cmd.Stdout = out
	if err := cmd.Run(); err != nil {
		return "", "", fmt.Errorf("%vError:\t%v\n", noConnectError, err)
	}
	dHost = strings.TrimSpace(out.String())
	log.WithField("host", dHost).Debug()

	// TODO: when go-dockerclient adds machine API endpoints use those instead.
	log.WithField("machine", machName).Debug("Querying Docker Machine cert path")
	cmd2 := exec.Command("docker-machine", "inspect", machName, "--format", "{{.HostOptions.AuthOptions.ServerCertPath}}")
	cmd2.Stdout = out2
	//cmd2.Stderr = os.Stderr
	if err := cmd2.Run(); err != nil {
		return "", "", fmt.Errorf("%vError:\t%v\n", noConnectError, err)
	}
	dPath = out2.String()
	dPath = strings.Replace(dPath, "'", "", -1)
	dPath = filepath.Dir(dPath)
	log.WithField("cert path", dPath).Debug()

	if dPath == "" || dHost == "" {
		return "", "", noConnectError
	}

	log.Info("Querying host and user have access to the right files for TLS connection to Docker")
	if err := checkKeysAndCerts(dPath); err != nil {
		return "", "", err
	}
	log.Debug("Certificate files look good")

	// technically, do not *have* to do this, but it will make repetitive tasks faster
	log.Debug("Setting environment variables for quick future development")
	os.Setenv("DOCKER_HOST", dHost)
	os.Setenv("DOCKER_CERT_PATH", dPath)
	os.Setenv("DOCKER_TLS_VERIFY", "1")
	os.Setenv("DOCKER_MACHINE_NAME", machName)

	log.WithField("machine", machName).Debug("Finished getting machine details")
	return dHost, dPath, nil
}

func setupErisMachine(driver string) error {
	cmd := "docker-machine"
	args := []string{"status", "eris"}
	if err := exec.Command(cmd, args...).Run(); err == nil {
		// if err == nil this means the machine is created. if err != nil that means machine doesn't exist.
		log.Debug("Eris Docker Machine exists. Starting")
		return startErisMachine()
	}
	log.Debug("Eris Docker Machine doesn't exist")

	return createErisMachine(driver)
}

func readActionDefinition(actionName []string, dropped map[string]string, varNum int) (*viper.Viper, map[string]string, error) {
	if len(actionName) == 0 {
		log.WithFields(log.Fields{
			"action": actionName,
			"drop":   dropped,
			"var#":   varNum,
		}).Debug("Failed to load action definition file")
		return nil, dropped, fmt.Errorf("The marmots could not find the action definition file.\nPlease check your actions with [eris actions ls]")
	}

	log.WithField("file", strings.Join(actionName, "_")).Debug("Preparing to read action definition file")
	log.WithField("drop", dropped).Debug()

	var actionConf = viper.New()

	actionConf.AddConfigPath(dir.ActionsPath)
	actionConf.SetConfigName(strings.Join(actionName, "_"))
	err := actionConf.ReadInConfig()

	if err != nil {
		log.WithField("action", actionName[len(actionName)-1]).Debug("Dropping and retrying")
		dropped[fmt.Sprintf("$%d", varNum)] = actionName[len(actionName)-1]
		actionName = actionName[:len(actionName)-1]
		varNum++
		return readActionDefinition(actionName, dropped, varNum)
	} else {
		log.Debug("Successfully read action definition file")
	}

	return actionConf, dropped, nil
}

func NewService(do *definitions.Do) error {
	srv := definitions.BlankServiceDefinition()
	srv.Name = do.Name
	srv.Service.Name = do.Name
	srv.Service.Image = do.Operations.Args[0]
	srv.Service.AutoData = true

	var err error
	//get maintainer info
	srv.Maintainer.Name, srv.Maintainer.Email, err = config.GitConfigUser()
	if err != nil {
		log.Debug(err.Error())
	}

	log.WithFields(log.Fields{
		"service": srv.Service.Name,
		"image":   srv.Service.Image,
	}).Debug("Creating a new service definition file")
	err = WriteServiceDefinitionFile(srv, filepath.Join(ServicesPath, do.Name+".toml"))
	if err != nil {
		return err
	}
	do.Result = "success"
	return nil
}

// this function is for parsing single variables
func printField(container interface{}, field string) error {
	log.WithField("=>", field).Debug("Inspecting field")
	var line string

	// We allow fields to be passed using dot syntax, but
	// we have to make sure all fields are Camelized
	lineSplit := strings.Split(field, ".")
	for n, f := range lineSplit {
		lineSplit[n] = camelize(f)
	}
	FieldCamel := strings.Join(lineSplit, ".")

	f, _ := reflections.GetFieldKind(container, FieldCamel)
	log.Debug("Field type", f)
	switch f.String() {
	case "ptr":
		//we don't recurse into to gain a bit more control... this function will be rarely used and doesn't have to be perfectly parseable.
	case "map":
		line = fmt.Sprintf("{{ range $key, $val := .%v }}{{ $key }}->{{ $val }}\n{{ end }}\n", FieldCamel)
	case "slice":
		line = fmt.Sprintf("{{ range .%v }}{{ . }}\n{{ end }}\n", FieldCamel)
	default:
		line = fmt.Sprintf("{{.%v}}\n", FieldCamel)
	}
	return writeTemplate(container, line)
}

//TODO test that this doesn't fmt things up, see note in #400
func moveOutOfDirAndRmDir(src, dest string) error {
	log.WithFields(log.Fields{
		"from": src,
		"to":   dest,
	}).Info("Move all files/dirs out of a dir and `rm -rf` that dir")
	toMove, err := filepath.Glob(filepath.Join(src, "*"))
	if err != nil {
		return err
	}

	if len(toMove) == 0 {
		log.Debug("No files to move")
	}

	for _, f := range toMove {
		// using a copy (read+write) strategy to get around swap partitions and other
		//   problems that cause a simple rename strategy to fail. it is more io overhead
		//   to do this, but for now that is preferable to alternative solutions.
		Copy(f, filepath.Join(dest, filepath.Base(f)))
	}

	log.WithField("=>", src).Info("Removing directory")
	err = os.RemoveAll(src)
	if err != nil {
		return err
	}

	return nil
}

func ManagePinned(do *definitions.Do) error {
	ensureRunning()
	if do.Rm && do.Hash != "" {
		return fmt.Errorf("Either remove a file by hash or all of them\n")
	}

	if do.Rm {
		log.Info("Removing all cached files")
		hashes, err := rmAllPinned()
		if err != nil {
			return err
		}
		do.Result = hashes
	} else if do.Hash != "" {
		log.WithField("hash", do.Hash).Info("Removing from cache")
		hashes, err := rmPinnedByHash(do.Hash)
		if err != nil {
			return err
		}
		do.Result = hashes
	} else {
		log.Debug("Listing files pinned locally")
		hash, err := listPinned()
		if err != nil {
			return err
		}
		do.Result = hash
	}
	return nil
}

func InstallEris() {
	goArgs := []string{"install", "./cmd/eris"}

	stdOut, err := exec.Command("go", goArgs...).CombinedOutput()
	if err != nil {
		log.Fatalf("Error with go install ./cmd/eris: %v", string(stdOut))
	}

	log.Debug("Go install worked correctly")
}

func startErisMachine() error {
	log.Info("Starting Eris Docker Machine")
	cmd := "docker-machine"
	args := []string{"start", "eris"}
	if err := exec.Command(cmd, args...).Run(); err != nil {
		return fmt.Errorf("There was an error starting the newly created docker-machine.\nError:\t%v\n", err)
	}
	log.Debug("Eris Docker Machine started")

	return nil
}

func canWeMigrate() bool {
	fmt.Print("Permission to migrate deprecated directories required: would you like to continue? (Y/y): ")
	var input string
	fmt.Scanln(&input)
	if input == "Y" || input == "y" || input == "YES" || input == "Yes" || input == "yes" {
		log.Debug("Confirmation verified. Proceeding")
		return true
	} else {
		return false
	}
}

func NewChain(do *definitions.Do) error {
	//overwrites directory if --force
	dir := filepath.Join(DataContainersPath, do.Name)
	if _, err := os.Stat(dir); err == nil {
		log.WithField("dir", dir).Debug("Chain data already exists in")
		if do.Force {
			log.Debug("Overwriting with new data")
			if os.RemoveAll(dir); err != nil {
				return err
			}
		} else {
			log.Debug("Using existing data; `--force` flag not given")
		}
	}

	// for now we just let setupChain force do.ChainID = do.Name
	// and we overwrite using jq in the container
	log.WithField("=>", do.Name).Debug("Setting up chain")
	return setupChain(do, loaders.ErisChainNew)
}

func PortsService(do *definitions.Do) error {
	service, err := loaders.LoadServiceDefinition(do.Name, false, do.Operations.ContainerNumber)
	if err != nil {
		return err
	}

	if IsServiceExisting(service.Service, service.Operations) {
		log.Debug("Service exists, getting port mapping")
		return util.PrintPortMappings(service.Operations.SrvContainerID, do.Operations.Args)
	}

	return nil
}

func createErisMachine(driver string) error {
	log.Warn("Creating the Eris Docker Machine")
	log.Warn("This will take some time, please feel free to go feed your marmot")
	log.WithField("driver", driver).Debug()
	cmd := "docker-machine"
	args := []string{"create", "--driver", driver, "eris"}
	if err := exec.Command(cmd, args...).Run(); err != nil {
		log.Debugf("There was an error creating the Eris Docker Machine: %v", err)
		return mustInstallError()
	}
	log.Debug("Eris Docker Machine created")

	return startErisMachine()
}

func connectDockerTLS(dockerHost, dockerCertPath string) error {
	var err error

	log.WithFields(log.Fields{
		"host":      dockerHost,
		"cert path": dockerCertPath,
	}).Debug("Connecting to Docker via TLS")
	DockerClient, err = docker.NewTLSClient(dockerHost, filepath.Join(dockerCertPath, "cert.pem"), filepath.Join(dockerCertPath, "key.pem"), filepath.Join(dockerCertPath, "ca.pem"))
	if err != nil {
		return err
	}

	log.Debug("Connected via TLS")
	return nil
}

func fixSteps(action *def.Action, dropReversed map[string]string) {
	log.Debug("Replacing $1, $2, $3 in steps with args from command line")

	if len(dropReversed) == 0 {
		return
	} else {
		log.WithField("replace", dropReversed).Debug()
	}

	// Because we pop from the end of the args list, the variables
	// in the map $1, $2, etc. are actually the exact opposite of
	// what they should be.
	dropped := make(map[string]string)
	j := 1
	for i := len(dropReversed); i > 0; i-- {
		log.WithField("=>", fmt.Sprintf("$%d:$%d", i, j)).Debug("Reversing")
		dropped[fmt.Sprintf("$%d", j)] = dropReversed[fmt.Sprintf("$%d", i)]
		j++
	}

	reg := regexp.MustCompile(`\$\d`)
	for n, step := range action.Steps {
		if reg.MatchString(step) {
			log.WithField("matched", step).Debug()
			for _, m := range reg.FindAllString(step, -1) {
				action.Steps[n] = strings.Replace(step, m, dropped[m], -1)
			}
			log.WithField("replaced", step).Debug()
		}
	}

	log.Debug("Checking fixed steps")
	for _, step := range action.Steps {
		log.WithField("step", step).Debug()
	}
}