func (s *systemtestSuite) verifyEPs(containers []*container) error {
	var err error

	// get the list of eps to verify
	epList := make([]string, len(containers))
	for ix, cont := range containers {
		epList[ix] = cont.eth0.ip
	}

	err = nil
	dbgOut := ""
	for try := 0; try < 20; try++ {
		for _, n := range s.nodes {
			if n.Name() == "k8master" {
				continue
			}
			dbgOut, err = n.verifyEPs(epList)
			if err != nil {
				break
			}
		}

		if err == nil {
			logrus.Info("EPs %+v verified on all nodes", epList)
			return nil
		}
		time.Sleep(1 * time.Second)
	}
	logrus.Errorf("Failed to verify EPs after 20 sec %v", err)
	logrus.Info("Debug output:\n %s", dbgOut)
	return err
}

// PrintRunningStep ...
func PrintRunningStep(title, version string, idx int) {
	if len(version) > 25 {
		version = "..." + stringutil.MaxLastChars(version, 22)
	}
	content := fmt.Sprintf("| (%d) %s (%s) |", idx, title, version)
	charDiff := len(content) - stepRunSummaryBoxWidthInChars

	if charDiff < 0 {
		// shorter than desired - fill with space
		content = fmt.Sprintf("| (%d) %s (%s)%s |", idx, title, version, strings.Repeat(" ", -charDiff))
	} else if charDiff > 0 {
		// longer than desired - trim title
		trimmedTitleWidth := len(title) - charDiff - 3
		if trimmedTitleWidth < 0 {
			log.Errorf("Step Version too long, can't present title at all! : %s", version)
		} else {
			content = fmt.Sprintf("| (%d) %s... (%s) |", idx, title[0:trimmedTitleWidth], version)
		}
	}

	sep := strings.Repeat("-", len(content))
	log.Info(sep)
	log.Infof(content)
	log.Info(sep)
	log.Info("|" + strings.Repeat(" ", stepRunSummaryBoxWidthInChars-2) + "|")
}

func setupLogging() {
	if config.Settings.Moire.Debug {
		log.Info("Logging in DEBUG mode")
		log.SetLevel(log.DebugLevel)
	}

	// log.SetFlags(log.Ldate | log.Ltime | log.Llongfile)
	log.SetFormatter(&log.TextFormatter{})

	if config.Settings.Moire.SentryDSN != "" {
		levels := []log.Level{
			log.PanicLevel,
			log.FatalLevel,
			log.ErrorLevel,
		}

		hook, err := logrus_sentry.NewSentryHook(config.Settings.Moire.SentryDSN, levels)
		if err != nil {
			log.Error("Unable to connect to sentry")
		} else {
			log.Info("Adding Sentry Hook")
			log.AddHook(hook)
		}
	}
}

// OptionKVOpts function returns an option setter for kvstore options
func OptionKVOpts(opts map[string]string) Option {
	return func(c *Config) {
		if opts["kv.cacertfile"] != "" && opts["kv.certfile"] != "" && opts["kv.keyfile"] != "" {
			log.Info("Option Initializing KV with TLS")
			tlsConfig, err := tlsconfig.Client(tlsconfig.Options{
				CAFile:   opts["kv.cacertfile"],
				CertFile: opts["kv.certfile"],
				KeyFile:  opts["kv.keyfile"],
			})
			if err != nil {
				log.Errorf("Unable to set up TLS: %s", err)
				return
			}
			if _, ok := c.Scopes[datastore.GlobalScope]; !ok {
				c.Scopes[datastore.GlobalScope] = &datastore.ScopeCfg{}
			}
			if c.Scopes[datastore.GlobalScope].Client.Config == nil {
				c.Scopes[datastore.GlobalScope].Client.Config = &store.Config{TLS: tlsConfig}
			} else {
				c.Scopes[datastore.GlobalScope].Client.Config.TLS = tlsConfig
			}
			// Workaround libkv/etcd bug for https
			c.Scopes[datastore.GlobalScope].Client.Config.ClientTLS = &store.ClientTLSConfig{
				CACertFile: opts["kv.cacertfile"],
				CertFile:   opts["kv.certfile"],
				KeyFile:    opts["kv.keyfile"],
			}
		} else {
			log.Info("Option Initializing KV without TLS")
		}
	}
}

// Start implementation of the tether.Extension interface
func (t *Toolbox) Start() error {
	t.Service.PrimaryIP = t.defaultIP

	t.stop = make(chan struct{})
	on := make(chan struct{})

	t.Service.PowerCommand.PowerOn.Handler = func() error {
		log.Info("toolbox: service is ready (power on event received)")
		close(on)
		return nil
	}

	err := t.Service.Start()
	if err != nil {
		return err
	}

	// Wait for the vmx to send the OS_PowerOn message,
	// at which point it will be ready to service vix command requests.
	log.Info("toolbox: waiting for initialization")

	select {
	case <-on:
	case <-time.After(time.Second):
		log.Warn("toolbox: timeout waiting for power on event")
	}

	return nil
}

func buildRedirectURL(core *roll.Core, w http.ResponseWriter, responseType, subject, scope string, app *roll.Application) (string, error) {
	log.Info("build redirect, app ctx: ", app.RedirectURI)

	var redirectURL string
	switch responseType {
	case "token":
		//Create signed token
		token, err := generateJWT(subject, scope, core, app)
		if err != nil {
			return "", err
		}
		redirectURL = fmt.Sprintf("%s#access_token=%s&token_type=Bearer", app.RedirectURI, token)
	case "code":
		token, err := generateSignedCode(core, subject, scope, app)
		if err != nil {
			return "", err
		}
		redirectURL = fmt.Sprintf("%s?code=%s", app.RedirectURI, token)
	default:
		panic(errors.New("unexpected response type in buildRedirectURL: " + responseType))
	}

	log.Info("redirect url: ", redirectURL)

	return redirectURL, nil
}

func runVault(docker *dockerclient.DockerClient) (string, string) {
	var bootedContainer bool

	//Is vault running?
	log.Info("Is vault running?")
	info := dockerutil.GetAcceptanceTestContainerInfo(docker, "atest-vault")

	if info != nil {
		log.Info("Vault container found - state is: ", info.State.StateString())
		log.Fatal("You must kill and remove the container manually - can't get the root token from an existing container in this test")
		return "", "" //not reached
	}

	log.Info("Vault is not running - create container context")
	bootedContainer = true
	vaultContainerCtx := createVaultTestContainerContext()

	//Create and start the container.
	log.Info("Create and start the container")
	containerId := dockerutil.CreateAndStartContainer(docker, []string{"IPC_LOCK"}, nil, vaultContainerCtx)

	if bootedContainer {
		//Give the container a little time to boot
		time.Sleep(1 * time.Second)
	}

	//Now initialize and unseal the damn vault
	rootToken := initializeVault()

	return containerId, rootToken
}

func (s *systemtestSuite) createVolume(host, tenant, name string, opts map[string]string) error {
	log.Infof("Creating %s/%s on %s", tenant, name, host)

	optsStr := []string{}

	if opts != nil {
		for key, value := range opts {
			optsStr = append(optsStr, "--opt")
			optsStr = append(optsStr, key+"="+value)
		}
	}

	cmd := fmt.Sprintf("docker volume create -d volplugin --name %s/%s %s", tenant, name, strings.Join(optsStr, " "))

	if out, err := s.vagrant.GetNode(host).RunCommandWithOutput(cmd); err != nil {
		log.Info(string(out))
		return err
	}

	if out, err := s.volcli(fmt.Sprintf("volume get %s %s", tenant, name)); err != nil {
		log.Error(out)
		return err
	}

	if out, err := s.vagrant.GetNode(host).RunCommandWithOutput(cmd); err != nil {
		log.Info(string(out))
		return err
	}

	return nil
}

func BuildConnection(ws *websocket.Conn) {
	log.Info("BuildConnection()")
	token := ws.Request().URL.Query().Get("token")
	log.Debug(token)
	var uci userConnectionInfo
	err := validateToken(token, time.Now(), ws.RemoteAddr(), &uci)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("validation error")
		// how should this reply to the client?
		return
	}

	onlineUser := &OnlineUser{
		Connection: ws,
		Uci:        uci,
		Send:       make(chan envelope, 256),
	}
	runningRoom.Users[onlineUser] = true
	go onlineUser.PushToClient()
	onlineUser.PullFromClient()
	delete(runningRoom.Users, onlineUser)
	log.Info("tore down user connection")
}

//NewLoadBalancer creates a new instance of a Round Robin load balancer
func (rrf *RoundRobinLoadBalancerFactory) NewLoadBalancer(backendName, caCertPath string, servers []config.ServerConfig) (LoadBalancer, error) {
	var rrlb RoundRobinLoadBalancer

	if backendName == "" {
		return nil, fmt.Errorf("Expected non-empty backend name")
	}

	if len(servers) == 0 {
		return nil, fmt.Errorf("Expected at least one server in servers argument")
	}

	rrlb.backend = backendName
	rrlb.servers = ring.New(len(servers))

	for _, s := range servers {

		lbEndpoint := new(LoadBalancerEndpoint)
		lbEndpoint.Address = fmt.Sprintf("%s:%d", s.Address, s.Port)
		metrics.SetGauge([]string{"endpoint", lbEndpoint.Address}, 1.0)
		lbEndpoint.PingURI = s.PingURI
		lbEndpoint.Up = true
		lbEndpoint.CACertPath = caCertPath

		log.Info("Spawing health check for address ", lbEndpoint.Address)
		healthCheckFunction := MakeHealthCheck(lbEndpoint, s, true)
		go healthCheckFunction()

		log.Info("Adding server with address ", lbEndpoint.Address)
		rrlb.servers.Value = lbEndpoint
		rrlb.servers = rrlb.servers.Next()
	}

	return &rrlb, nil
}

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestFormat1TestSuite(t *testing.T) {
	//ts := new(Format1TestSuite)
	log.Info("TestFormat1TestSuite - Running test suite")
	suite.Run(t, new(Format1TestSuite))
	log.Info("TestFormat1TestSuite - Finished test suite")

}

func importCountries(conn *transaction.Connection, config model.YAMLCountries) (err error) {
	log.Info("Import countries started")

	if err = conn.RemoveCountriesTranslations(); err != nil {
		return
	}
	for countryCode, country := range config.Countries {
		if conn.IsCountryExist(countryCode) == false {
			err = errors.New("Missing country")
			break
		}

		errorInsideTranslation := false
		for languageCode, translation := range country.Translations {
			if conn.IsLanguageExist(languageCode) == false {
				errorInsideTranslation = true
				err = errors.New("No language")
				break
			}

			if err = conn.CreateCountryTranslation(countryCode, languageCode, translation); err != nil {
				errorInsideTranslation = true
				break
			}
		}

		if errorInsideTranslation == true {
			break
		}
	}
	log.Info("Import countries finished")
	return
}

func importPhoneNumbersCategories(conn *transaction.Connection, config model.YAMLPhoneNumbersCategories) (err error) {
	log.Info("Import phonenumber categories started")

	if err = conn.RemovePhonenumbersAndCategories(); err != nil {
		return
	}

	for categoryCode, category := range config.Categories {
		if err = conn.CreatePhonenumberCategory(categoryCode); err != nil {
			break
		}

		errorInsideTranslation := false
		for languageCode, translation := range category.Translations {
			if conn.IsLanguageExist(languageCode) == false {
				errorInsideTranslation = true
				err = errors.New("No language")
				break
			}

			if err = conn.CreatePhonenumberCategoryTranslation(categoryCode, languageCode, translation); err != nil {
				errorInsideTranslation = true
				break
			}
		}

		if errorInsideTranslation == true {
			break
		}
	}
	log.Info("Import phonenumber categories finished")
	return
}

func (r *Runner) getVersions(options *flags.Options) (siVersion, cloudVersion int, err error) {
	var siError, cloudError error
	versionChan := make(chan struct{}, 2)

	go func() {
		siVersion, siError = r.imgDataOrigin.GetLatestVersion(options)
		log.Info("siVersion: ", siVersion)
		versionChan <- struct{}{}
	}()

	go func() {
		cloudVersion, cloudError = r.imgDataTarget.GetLatestVersion(options)
		log.Info("cloudVersion: ", cloudVersion)
		versionChan <- struct{}{}
	}()

	for i := 0; i < 2; i++ {
		<-versionChan
	}

	if siError != nil {
		return 0, 0, siError
	}
	if cloudError != nil {
		if _, ok := cloudError.(*cloud.ErrVersionNotFound); !ok {
			return 0, 0, cloudError
		}
	}
	return
}

//GetManifestsV2Handler is
func GetManifestsV2Handler(ctx *macaron.Context) (int, []byte) {
	repository := ctx.Params(":repository")
	namespace := ctx.Params(":namespace")
	tag := ctx.Params(":tag")

	t := new(models.DockerTagV2)
	if _, err := t.Get(namespace, repository, tag); err != nil && err == gorm.ErrRecordNotFound {
		log.Info("Not found repository in tetting tag manifest: %s/%s:%s", namespace, repository, tag)

		result, _ := module.EncodingError(module.BLOB_UNKNOWN, fmt.Sprintf("%s/%s", namespace, repository))
		return http.StatusNotFound, result
	} else if err != nil && err != gorm.ErrRecordNotFound {
		log.Info("Failed to get tag manifest %s/%s:%s : ", namespace, repository, tag, err.Error())

		result, _ := module.EncodingError(module.UNKNOWN, err.Error())
		return http.StatusBadRequest, result
	}

	digest, _ := signature.DigestManifest([]byte(t.Manifest))

	ctx.Resp.Header().Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json")
	ctx.Resp.Header().Set("Docker-Content-Digest", digest)
	ctx.Resp.Header().Set("Content-Length", fmt.Sprint(len(t.Manifest)))

	return http.StatusOK, []byte(t.Manifest)
}

// we'll connect to the database through this function
func Init() {
	dbMutex.Lock()
	defer dbMutex.Unlock()

	if initialized {
		return
	}

	DBUrl = url.URL{
		Scheme:   "postgres",
		Host:     config.Constants.DbAddr,
		Path:     config.Constants.DbDatabase,
		RawQuery: "sslmode=disable",
	}

	logrus.Info("Connecting to DB on %s", DBUrl.String())

	DBUrl.User = url.UserPassword(config.Constants.DbUsername, config.Constants.DbPassword)

	var err error
	DB, err = gorm.Open("postgres", DBUrl.String())
	//	DB, err := gorm.Open("sqlite3", "/tmp/gorm.db")
	if err != nil {
		logrus.Fatal(err.Error())
	}

	DB.SetLogger(logrus.StandardLogger())

	logrus.Info("Connected!")
	initialized = true
}

//HeadBlobsV2Handler is
func HeadBlobsV2Handler(ctx *macaron.Context) (int, []byte) {
	digest := ctx.Params(":digest")
	tarsum := strings.Split(digest, ":")[1]

	i := new(models.DockerImageV2)
	if err := i.Get(tarsum); err != nil && err == gorm.ErrRecordNotFound {
		log.Info("Not found blob: %s", tarsum)

		result, _ := module.EncodingError(module.BLOB_UNKNOWN, digest)
		return http.StatusNotFound, result
	} else if err != nil && err != gorm.ErrRecordNotFound {
		log.Info("Failed to get blob %s: %s", tarsum, err.Error())

		result, _ := module.EncodingError(module.UNKNOWN, err.Error())
		return http.StatusBadRequest, result
	}

	ctx.Resp.Header().Set("Content-Type", "application/json; charset=utf-8")
	ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
	ctx.Resp.Header().Set("Docker-Content-Digest", digest)
	ctx.Resp.Header().Set("Content-Length", fmt.Sprint(i.Size))

	result, _ := json.Marshal(map[string]string{})
	return http.StatusOK, result
}

func main() {
	log.Info("GlusterD starting")

	context.Init()

	for _, c := range commands.Commands {
		context.Rest.SetRoutes(c.Routes())
	}

	sigCh := make(chan os.Signal)
	signal.Notify(sigCh)
	go func() {
		for s := range sigCh {
			log.WithField("signal", s).Debug("Signal recieved")
			switch s {
			case os.Interrupt:
				log.WithField("signal", s).Info("Recieved SIGTERM. Stopping GlusterD.")
				context.Rest.Stop()
				log.Info("Termintaing GlusterD.")
				os.Exit(0)

			default:
				continue
			}
		}
	}()

	err := context.Rest.Listen()
	if err != nil {
		log.Fatal("Could not start GlusterD Rest Server. Aborting.")
	}
}

func (d *DBConf) init() {

	// get the schema, and apply it before moving on
	s := version.GetSchemaFromDb(d.db)

	// bootstrap
	if s.Version == version.First {
		logrus.Info("Bootstrapping config db")
		err := s.Apply(d.db)
		if err != nil {
			logrus.Errorf("Unable to bootstrap config %s", err.Error())
		}

	}

	// if we don't have the correct version, apply the new one
	if version.LatestSchema().Greater(s) {
		logrus.Info("Upgrading config db version from %s to %s", s.Version, version.LatestSchema().Version)
		err := version.LatestSchema().Apply(d.db)
		if err != nil {
			logrus.Errorf("Unable to apply schema version %s to config db %s", version.LatestSchema().Version, err.Error())
		}
	}

	logrus.Infof("Using db config version %s", s.Version)

	err := d.initAdminUser()
	if err != nil {
		logrus.Errorf("Unable init admin user: %s", err)
	}
}

func (pm *ProcessManager) background(waitChan chan pidChangeNotification, subscriptions []chan int, signals chan os.Signal) {
	log.Info("Going into background mode")
	for {
		select {
		case status := <-waitChan:
			{
				pm.notify(status.wstatus.ExitStatus(), subscriptions)
				pm.tdcb()
				pm.killProcess(waitChan)
				return
			}
		case <-signals:
			{
				log.Info("Tearing down at signal")
				pm.notify(-1, subscriptions)
				pm.tdcb()
				pm.killProcess(waitChan)
				return
			}
		case tearDownChan := <-pm.teardown:
			{
				log.Info("Tearing down")
				pm.notify(-1, subscriptions)
				pm.killProcess(waitChan)
				pm.tdcb()
				tearDownChan <- nil
				return
			}
		case subscribe := <-pm.subscribe:
			{
				subscriptions = append(subscriptions, subscribe)
			}
		}
	}
}