func newObcBatch(id uint64, config *viper.Viper, stack consensus.Stack) *obcBatch {
	var err error

	op := &obcBatch{
		obcGeneric: obcGeneric{stack},
		stack:      stack,
	}

	op.persistForward.persistor = stack

	logger.Debug("Replica %d obtaining startup information", id)

	op.pbft = newPbftCore(id, config, op)

	op.batchSize = config.GetInt("general.batchSize")
	op.batchStore = nil
	op.batchTimeout, err = time.ParseDuration(config.GetString("general.timeout.batch"))
	if err != nil {
		panic(fmt.Errorf("Cannot parse batch timeout: %s", err))
	}

	op.incomingChan = make(chan *batchMessage)

	// create non-running timer
	op.batchTimer = time.NewTimer(100 * time.Hour) // XXX ugly
	op.batchTimer.Stop()

	op.idleChan = make(chan struct{})

	go op.main()
	return op
}

func setDefaultConfigValuesWithViper(v *viper.Viper, b *BgpConfigSet) error {
	if v == nil {
		v = viper.New()
	}

	if err := SetDefaultGlobalConfigValues(&b.Global); err != nil {
		return err
	}

	for idx, server := range b.BmpServers {
		if server.Config.Port == 0 {
			server.Config.Port = bmp.BMP_DEFAULT_PORT
		}
		b.BmpServers[idx] = server
	}

	if b.Zebra.Config.Url == "" {
		b.Zebra.Config.Url = "unix:/var/run/quagga/zserv.api"
	}

	list, err := extractArray(v.Get("neighbors"))
	if err != nil {
		return err
	}

	for idx, n := range b.Neighbors {
		vv := viper.New()
		if len(list) > idx {
			vv.Set("neighbor", list[idx])
		}
		if err := setDefaultNeighborConfigValuesWithViper(vv, &n, b.Global.Config.As); err != nil {
			return err
		}
		b.Neighbors[idx] = n
	}

	for idx, r := range b.RpkiServers {
		if r.Config.Port == 0 {
			b.RpkiServers[idx].Config.Port = rtr.RPKI_DEFAULT_PORT
		}
	}

	list, err = extractArray(v.Get("policy-definitions"))
	if err != nil {
		return err
	}

	for idx, p := range b.PolicyDefinitions {
		vv := viper.New()
		if len(list) > idx {
			vv.Set("policy", list[idx])
		}
		if err := setDefaultPolicyConfigValuesWithViper(vv, &p); err != nil {
			return err
		}
		b.PolicyDefinitions[idx] = p
	}

	return nil
}

func (k *keyCommander) getKeyStores(
	config *viper.Viper, withHardware, hardwareBackup bool) ([]trustmanager.KeyStore, error) {

	retriever := k.getRetriever()

	directory := config.GetString("trust_dir")
	fileKeyStore, err := trustmanager.NewKeyFileStore(directory, retriever)
	if err != nil {
		return nil, fmt.Errorf(
			"Failed to create private key store in directory: %s", directory)
	}

	ks := []trustmanager.KeyStore{fileKeyStore}

	if withHardware {
		var yubiStore trustmanager.KeyStore
		if hardwareBackup {
			yubiStore, err = getYubiStore(fileKeyStore, retriever)
		} else {
			yubiStore, err = getYubiStore(nil, retriever)
		}
		if err == nil && yubiStore != nil {
			// Note that the order is important, since we want to prioritize
			// the yubikey store
			ks = []trustmanager.KeyStore{yubiStore, fileKeyStore}
		}
	}

	return ks, nil
}

func NewOBCExecutor(config *viper.Viper, orderer Orderer, stack statetransfer.PartialStack) (obcex *obcExecutor) {
	var err error
	obcex = &obcExecutor{}
	queueSize := config.GetInt("executor.queuesize")
	if queueSize <= 0 {
		panic("Queue size must be positive")
	}

	obcex.executionQueue = make(chan *transaction, queueSize)
	obcex.syncTargets = make(chan *syncTarget)
	obcex.completeSync = make(chan *syncTarget)
	obcex.threadIdle = make(chan struct{})
	obcex.threadExit = make(chan struct{})
	obcex.orderer = orderer
	obcex.executorStack = stack
	obcex.id, _, err = stack.GetNetworkHandles()
	if nil != err {
		logger.Error("Could not resolve our own PeerID, assigning dummy")
		obcex.id = &pb.PeerID{"Dummy"}
	}

	logger.Info("Executor for %v using queue size of %d", obcex.id, queueSize)

	obcex.sts = statetransfer.NewStateTransferState(config, stack)

	listener := struct{ statetransfer.ProtoListener }{}
	listener.CompletedImpl = obcex.stateTransferCompleted
	obcex.sts.RegisterListener(&listener)

	go obcex.queueThread()
	return
}

func newObcBatch(id uint64, config *viper.Viper, stack consensus.Stack) *obcBatch {
	var err error

	op := &obcBatch{
		obcGeneric: obcGeneric{stack: stack},
	}

	op.persistForward.persistor = stack

	logger.Debugf("Replica %d obtaining startup information", id)

	op.manager = events.NewManagerImpl() // TODO, this is hacky, eventually rip it out
	op.manager.SetReceiver(op)
	etf := events.NewTimerFactoryImpl(op.manager)
	op.pbft = newPbftCore(id, config, op, etf)
	op.manager.Start()
	blockchainInfoBlob := stack.GetBlockchainInfoBlob()
	op.externalEventReceiver.manager = op.manager
	op.broadcaster = newBroadcaster(id, op.pbft.N, op.pbft.f, op.pbft.broadcastTimeout, stack)
	op.manager.Queue() <- workEvent(func() {
		op.pbft.stateTransfer(&stateUpdateTarget{
			checkpointMessage: checkpointMessage{
				seqNo: op.pbft.lastExec,
				id:    blockchainInfoBlob,
			},
		})
	})

	op.batchSize = config.GetInt("general.batchsize")
	op.batchStore = nil
	op.batchTimeout, err = time.ParseDuration(config.GetString("general.timeout.batch"))
	if err != nil {
		panic(fmt.Errorf("Cannot parse batch timeout: %s", err))
	}
	logger.Infof("PBFT Batch size = %d", op.batchSize)
	logger.Infof("PBFT Batch timeout = %v", op.batchTimeout)

	if op.batchTimeout >= op.pbft.requestTimeout {
		op.pbft.requestTimeout = 3 * op.batchTimeout / 2
		logger.Warningf("Configured request timeout must be greater than batch timeout, setting to %v", op.pbft.requestTimeout)
	}

	if op.pbft.requestTimeout >= op.pbft.nullRequestTimeout && op.pbft.nullRequestTimeout != 0 {
		op.pbft.nullRequestTimeout = 3 * op.pbft.requestTimeout / 2
		logger.Warningf("Configured null request timeout must be greater than request timeout, setting to %v", op.pbft.nullRequestTimeout)
	}

	op.incomingChan = make(chan *batchMessage)

	op.batchTimer = etf.CreateTimer()

	op.reqStore = newRequestStore()

	op.deduplicator = newDeduplicator()

	op.idleChan = make(chan struct{})
	close(op.idleChan) // TODO remove eventually

	return op
}

func collectFlags(v *viper.Viper, cmd *cobra.Command) {
	v.BindPFlags(cmd.PersistentFlags())
	v.BindPFlags(cmd.Flags())
	for _, cmd := range cmd.Commands() {
		collectFlags(v, cmd)
	}
}

// Parse the cache configurations for GET-ting current and checksummed metadata,
// returning the configuration for current (non-content-addressed) metadata
// first, then the configuration for consistent (content-addressed) metadata
// second. The configuration consists mainly of the max-age (an integer in seconds,
// just like in the Cache-Control header) for each type of metadata.
// The max-age must be between 0 and 31536000 (one year in seconds, which is
// the recommended maximum time data is cached), else parsing will return an error.
// A max-age of 0 will disable caching for that type of download (consistent or current).
func getCacheConfig(configuration *viper.Viper) (current, consistent utils.CacheControlConfig, err error) {
	cccs := make(map[string]utils.CacheControlConfig)
	currentOpt, consistentOpt := "current_metadata", "consistent_metadata"

	defaults := map[string]int{
		currentOpt:    int(notary.CurrentMetadataCacheMaxAge.Seconds()),
		consistentOpt: int(notary.ConsistentMetadataCacheMaxAge.Seconds()),
	}
	maxMaxAge := int(notary.CacheMaxAgeLimit.Seconds())

	for optionName, seconds := range defaults {
		m := configuration.GetString(fmt.Sprintf("caching.max_age.%s", optionName))
		if m != "" {
			seconds, err = strconv.Atoi(m)
			if err != nil || seconds < 0 || seconds > maxMaxAge {
				return nil, nil, fmt.Errorf(
					"must specify a cache-control max-age between 0 and %v", maxMaxAge)
			}
		}
		cccs[optionName] = utils.NewCacheControlConfig(seconds, optionName == currentOpt)
	}
	current = cccs[currentOpt]
	consistent = cccs[consistentOpt]
	return
}

func makePBFTNetwork(N int, config *viper.Viper) *pbftNetwork {
	if config == nil {
		config = loadConfig()
	}

	config.Set("general.N", N)
	config.Set("general.f", (N-1)/3)
	endpointFunc := func(id uint64, net *testnet) endpoint {
		tep := makeTestEndpoint(id, net)
		pe := &pbftEndpoint{
			testEndpoint: tep,
			manager:      events.NewManagerImpl(),
		}

		pe.sc = &simpleConsumer{
			pe: pe,
		}

		pe.pbft = newPbftCore(id, config, pe.sc, events.NewTimerFactoryImpl(pe.manager))
		pe.manager.SetReceiver(pe.pbft)

		pe.manager.Start()

		return pe

	}

	pn := &pbftNetwork{testnet: makeTestnet(N, endpointFunc)}
	pn.pbftEndpoints = make([]*pbftEndpoint, len(pn.endpoints))
	for i, ep := range pn.endpoints {
		pn.pbftEndpoints[i] = ep.(*pbftEndpoint)
		pn.pbftEndpoints[i].sc.pbftNet = pn
	}
	return pn
}

// ParseStorage tries to parse out Storage from a Viper.  If backend and
// URL are not provided, returns a nil pointer.  Storage is required (if
// a backend is not provided, an error will be returned.)
func ParseStorage(configuration *viper.Viper, allowedBackends []string) (*Storage, error) {
	store := Storage{
		Backend: configuration.GetString("storage.backend"),
		Source:  configuration.GetString("storage.db_url"),
	}

	supported := false
	store.Backend = strings.ToLower(store.Backend)
	for _, backend := range allowedBackends {
		if backend == store.Backend {
			supported = true
			break
		}
	}

	if !supported {
		return nil, fmt.Errorf(
			"must specify one of these supported backends: %s",
			strings.Join(allowedBackends, ", "))
	}

	if store.Backend == MemoryBackend {
		return &Storage{Backend: MemoryBackend}, nil
	}
	if store.Source == "" {
		return nil, fmt.Errorf(
			"must provide a non-empty database source for %s", store.Backend)
	}
	return &store, nil
}

func getKeysRecursively(base string, v *viper.Viper, nodeKeys map[string]interface{}) map[string]interface{} {
	result := make(map[string]interface{})
	for key := range nodeKeys {
		fqKey := base + key
		val := v.Get(fqKey)
		if m, ok := val.(map[interface{}]interface{}); ok {
			logger.Debugf("Found map[interface{}]interface{} value for %s", fqKey)
			tmp := make(map[string]interface{})
			for ik, iv := range m {
				cik, ok := ik.(string)
				if !ok {
					panic("Non string key-entry")
				}
				tmp[cik] = iv
			}
			result[key] = getKeysRecursively(fqKey+".", v, tmp)
		} else if m, ok := val.(map[string]interface{}); ok {
			logger.Debugf("Found map[string]interface{} value for %s", fqKey)
			result[key] = getKeysRecursively(fqKey+".", v, m)
		} else {
			logger.Debugf("Found real value for %s setting to %T %v", fqKey, val, val)
			result[key] = val
		}
	}
	return result
}

func maybeAutoPublish(cmd *cobra.Command, doPublish bool, gun string, config *viper.Viper, passRetriever notary.PassRetriever) error {

	if !doPublish {
		return nil
	}

	// We need to set up a http RoundTripper when publishing
	rt, err := getTransport(config, gun, readWrite)
	if err != nil {
		return err
	}

	trustPin, err := getTrustPinning(config)
	if err != nil {
		return err
	}

	nRepo, err := notaryclient.NewFileCachedNotaryRepository(
		config.GetString("trust_dir"), gun, getRemoteTrustServer(config), rt, passRetriever, trustPin)
	if err != nil {
		return err
	}

	cmd.Println("Auto-publishing changes to", gun)
	return publishAndPrintToCLI(cmd, nRepo, gun)
}

// Open and return a listening unix socket.
func NewUnixListener(v *viper.Viper) (*net.UnixListener, error) {
	l, err := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: v.GetString("socket")})
	if err != nil {
		return nil, err
	}

	return l, nil
}

// GetPathRelativeToConfig gets a configuration key which is a path, and if
// it is not empty or an absolute path, returns the absolute path relative
// to the configuration file
func GetPathRelativeToConfig(configuration *viper.Viper, key string) string {
	configFile := configuration.ConfigFileUsed()
	p := configuration.GetString(key)
	if p == "" || filepath.IsAbs(p) {
		return p
	}
	return filepath.Clean(filepath.Join(filepath.Dir(configFile), p))
}

func loadBoolPtr(key string, v *viper.Viper) *bool {
	val := v.Get(key)
	if val == nil {
		return nil
	}
	b := v.GetBool(key)
	return &b
}

func AddDefaults(c *viper.Viper) {
	c.SetDefault("marathon", kv("host", "http://localhost:8080"))
	c.SetDefault("mesos", kv("master", "localhost:5050"))
	c.SetDefault("zk", kv("host", "localhost:2181"))

	cachePath, _ := homedir.Expand("~/.marathonctl/packages")
	c.SetDefault("package-cache-path", cachePath)
	c.SetDefault("package-repo", "github.com/ashwanthkumar/marathonctl-universe")
}

func newInstance(v *viper.Viper) *instance {
	var i = instance{
		viper: v,
		slots: v.GetInt("start.parallel"),
	}
	i.slotAvailable = sync.NewCond(&i.m)
	i.taskFinished = sync.NewCond(&i.m)
	return &i
}

// New is the constructor for Application struct.
func New(config *viper.Viper) (*Application, error) {
	cookieStoreSecret := config.Get("cookie_secret").(string)

	app := &Application{}
	app.config = config
	app.cookieStore = sessions.NewCookieStore([]byte(cookieStoreSecret))

	return app, nil
}

// ParseLogLevel tries to parse out a log level from a Viper.  If there is no
// configuration, defaults to the provided error level
func ParseLogLevel(configuration *viper.Viper, defaultLevel logrus.Level) (
	logrus.Level, error) {

	logStr := configuration.GetString("logging.level")
	if logStr == "" {
		return defaultLevel, nil
	}
	return logrus.ParseLevel(logStr)
}

// gets the required gun prefixes accepted by this server
func getRequiredGunPrefixes(configuration *viper.Viper) ([]string, error) {
	prefixes := configuration.GetStringSlice("repositories.gun_prefixes")
	for _, prefix := range prefixes {
		p := path.Clean(strings.TrimSpace(prefix))
		if p+"/" != prefix || strings.HasPrefix(p, "/") || strings.HasPrefix(p, "..") {
			return nil, fmt.Errorf("invalid GUN prefix %s", prefix)
		}
	}
	return prefixes, nil
}

func newPbftCore(id uint64, config *viper.Viper, consumer innerCPI) *pbftCore {
	instance := &pbftCore{}
	instance.id = id
	instance.consumer = consumer

	// in dev/debugging mode you are expected to override the config values
	// with the environment variable OPENCHAIN_OBCPBFT_X_Y

	// read from the config file
	// you can override the config values with the environment variable prefix
	// OPENCHAIN_OBCPBFT, e.g. OPENCHAIN_OBCPBFT_BYZANTINE
	var err error
	instance.byzantine = config.GetBool("replica.byzantine")
	instance.f = config.GetInt("general.f")
	instance.K = uint64(config.GetInt("general.K"))
	instance.requestTimeout, err = time.ParseDuration(config.GetString("general.timeout.request"))
	if err != nil {
		panic(fmt.Errorf("Cannot parse request timeout: %s", err))
	}
	instance.newViewTimeout, err = time.ParseDuration(config.GetString("general.timeout.viewchange"))
	if err != nil {
		panic(fmt.Errorf("Cannot parse new view timeout: %s", err))
	}

	instance.activeView = true
	instance.L = 2 * instance.K // log size
	instance.replicaCount = 3*instance.f + 1

	// init the logs
	instance.certStore = make(map[msgID]*msgCert)
	instance.reqStore = make(map[string]*Request)
	instance.checkpointStore = make(map[Checkpoint]bool)
	instance.chkpts = make(map[uint64]string)
	instance.viewChangeStore = make(map[vcidx]*ViewChange)
	instance.pset = make(map[uint64]*ViewChange_PQ)
	instance.qset = make(map[qidx]*ViewChange_PQ)
	instance.newViewStore = make(map[uint64]*NewView)

	// load genesis checkpoint
	stateHash, err := instance.consumer.getStateHash(0)
	if err != nil {
		panic(fmt.Errorf("Cannot load genesis block: %s", err))
	}
	instance.chkpts[0] = base64.StdEncoding.EncodeToString(stateHash)

	// create non-running timer XXX ugly
	instance.newViewTimer = time.NewTimer(100 * time.Hour)
	instance.newViewTimer.Stop()
	instance.lastNewViewTimeout = instance.newViewTimeout
	instance.outstandingReqs = make(map[string]*Request)

	go instance.timerHander()

	return instance
}