func TestNoPersist(t *testing.T) {
	cfgOptions, err := OptionBoltdbWithRandomDBFile()
	if err != nil {
		t.Fatalf("Error creating random boltdb file : %v", err)
	}
	ctrl, err := New(cfgOptions...)
	if err != nil {
		t.Fatalf("Error new controller: %v", err)
	}
	nw, err := ctrl.NewNetwork("host", "host", NetworkOptionPersist(false))
	if err != nil {
		t.Fatalf("Error creating default \"host\" network: %v", err)
	}
	ep, err := nw.CreateEndpoint("newendpoint", []EndpointOption{}...)
	if err != nil {
		t.Fatalf("Error creating endpoint: %v", err)
	}
	store := ctrl.(*controller).getStore(datastore.LocalScope).KVStore()
	if exists, _ := store.Exists(datastore.Key(datastore.NetworkKeyPrefix, string(nw.ID()))); exists {
		t.Fatalf("Network with persist=false should not be stored in KV Store")
	}
	if exists, _ := store.Exists(datastore.Key([]string{datastore.EndpointKeyPrefix, string(nw.ID()), string(ep.ID())}...)); exists {
		t.Fatalf("Endpoint in Network with persist=false should not be stored in KV Store")
	}
	store.Close()
}

func (c *controller) getNetworksFromStore() ([]*network, error) {
	var nl []*network

	for _, store := range c.getStores() {
		kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
			&network{ctrlr: c})
		// Continue searching in the next store if no keys found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				log.Debugf("failed to get networks for scope %s: %v", store.Scope(), err)
			}
			continue
		}

		for _, kvo := range kvol {
			n := kvo.(*network)
			n.Lock()
			n.ctrlr = c
			n.Unlock()

			ec := &endpointCnt{n: n}
			err = store.GetObject(datastore.Key(ec.Key()...), ec)
			if err != nil {
				return nil, fmt.Errorf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
			}

			n.epCnt = ec
			n.scope = store.Scope()
			nl = append(nl, n)
		}
	}

	return nl, nil
}

func (c *controller) getNetworkFromStore(nid string) (*network, error) {
	for _, store := range c.getStores() {
		n := &network{id: nid, ctrlr: c}
		err := store.GetObject(datastore.Key(n.Key()...), n)
		// Continue searching in the next store if the key is not found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				log.Debugf("could not find network %s: %v", nid, err)
			}
			continue
		}

		ec := &endpointCnt{n: n}
		err = store.GetObject(datastore.Key(ec.Key()...), ec)
		if err != nil {
			return nil, fmt.Errorf("could not find endpoint count for network %s: %v", n.Name(), err)
		}

		n.epCnt = ec
		n.scope = store.Scope()
		return n, nil
	}

	return nil, fmt.Errorf("network %s not found", nid)
}

func (c *controller) getNetworksForScope(scope string) ([]*network, error) {
	var nl []*network

	store := c.getStore(scope)
	if store == nil {
		return nil, nil
	}

	kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
		&network{ctrlr: c})
	if err != nil && err != datastore.ErrKeyNotFound {
		return nil, fmt.Errorf("failed to get networks for scope %s: %v",
			scope, err)
	}

	for _, kvo := range kvol {
		n := kvo.(*network)
		n.ctrlr = c

		ec := &endpointCnt{n: n}
		err = store.GetObject(datastore.Key(ec.Key()...), ec)
		if err != nil {
			log.Warnf("Could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
			continue
		}

		n.epCnt = ec
		n.scope = scope
		nl = append(nl, n)
	}

	return nl, nil
}

func (n *network) watchEndpoints() error {
	if n.Skip() || !n.ctrlr.validateGlobalStoreConfig() {
		return nil
	}

	n.Lock()
	cs := n.ctrlr.globalStore
	tmp := endpoint{network: n}
	n.stopWatchCh = make(chan struct{})
	stopCh := n.stopWatchCh
	n.Unlock()

	endpointKey := datastore.Key(tmp.KeyPrefix()...)
	if err := ensureKeys(endpointKey, cs); err != nil {
		return fmt.Errorf("failed to ensure if the endpoint keys are valid and present in store: %v", err)
	}
	epPairs, err := cs.KVStore().WatchTree(endpointKey, stopCh)
	if err != nil {
		return err
	}
	go func() {
		for {
			select {
			case <-stopCh:
				return
			case eps := <-epPairs:
				n.Lock()
				tmpview := endpointTable{}
				lview := n.endpoints
				n.Unlock()
				for k, v := range lview {
					if v.network.isGlobalScoped() {
						tmpview[k] = v
					}
				}
				n.ctrlr.processEndpointsUpdate(eps, &tmpview)
				// Delete processing
				for k := range tmpview {
					n.Lock()
					existing, ok := n.endpoints[k]
					n.Unlock()
					if !ok {
						continue
					}
					tmp := endpoint{}
					if err := cs.GetObject(datastore.Key(existing.Key()...), &tmp); err != datastore.ErrKeyNotFound {
						continue
					}
					if err := existing.deleteEndpoint(); err != nil {
						log.Debugf("Delete failed %s: %s", existing.name, err)
					}
				}
			}
		}
	}()
	return nil
}

func (c *controller) watchNetworks() error {
	if !c.validateDatastoreConfig() {
		return nil
	}

	c.Lock()
	cs := c.store
	c.Unlock()

	networkKey := datastore.Key(datastore.NetworkKeyPrefix)
	if err := ensureKeys(networkKey, cs); err != nil {
		return fmt.Errorf("failed to ensure if the network keys are valid and present in store: %v", err)
	}
	nwPairs, err := cs.KVStore().WatchTree(networkKey, nil)
	if err != nil {
		return err
	}
	go func() {
		for {
			select {
			case nws := <-nwPairs:
				c.Lock()
				tmpview := networkTable{}
				lview := c.networks
				c.Unlock()
				for k, v := range lview {
					global, _ := v.isGlobalScoped()
					if global {
						tmpview[k] = v
					}
				}
				c.processNetworkUpdate(nws, &tmpview)

				// Delete processing
				for k := range tmpview {
					c.Lock()
					existing, ok := c.networks[k]
					c.Unlock()
					if !ok {
						continue
					}
					tmp := network{}
					if err := c.store.GetObject(datastore.Key(existing.Key()...), &tmp); err != datastore.ErrKeyNotFound {
						continue
					}
					if err := existing.deleteNetwork(); err != nil {
						log.Debugf("Delete failed %s: %s", existing.name, err)
					}
				}
			}
		}
	}()
	return nil
}

func (d *driver) populateEndpoints() error {
	kvol, err := d.store.List(datastore.Key(bridgeEndpointPrefix), &bridgeEndpoint{})
	if err != nil && err != datastore.ErrKeyNotFound {
		return fmt.Errorf("failed to get bridge endpoints from store: %v", err)
	}

	if err == datastore.ErrKeyNotFound {
		return nil
	}

	for _, kvo := range kvol {
		ep := kvo.(*bridgeEndpoint)
		n, ok := d.networks[ep.nid]
		if !ok {
			logrus.Debugf("Network (%s) not found for restored bridge endpoint (%s)", ep.nid[0:7], ep.id[0:7])
			logrus.Debugf("Deleting stale bridge endpoint (%s) from store", ep.nid[0:7])
			if err := d.storeDelete(ep); err != nil {
				logrus.Debugf("Failed to delete stale bridge endpoint (%s) from store", ep.nid[0:7])
			}
			continue
		}
		n.endpoints[ep.id] = ep
		n.restorePortAllocations(ep)
		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])
	}

	return nil
}

func (ec *endpointCnt) atomicIncDecEpCnt(inc bool) error {
retry:
	ec.Lock()
	if inc {
		ec.Count++
	} else {
		ec.Count--
	}
	ec.Unlock()

	store := ec.n.getController().getStore(ec.DataScope())
	if store == nil {
		return fmt.Errorf("store not found for scope %s", ec.DataScope())
	}

	if err := ec.n.getController().updateToStore(ec); err != nil {
		if err == datastore.ErrKeyModified {
			if err := store.GetObject(datastore.Key(ec.Key()...), ec); err != nil {
				return fmt.Errorf("could not update the kvobject to latest when trying to atomic add endpoint count: %v", err)
			}

			goto retry
		}

		return err
	}

	return nil
}

// NewHandle returns a thread-safe instance of the bitmask handler
func NewHandle(app string, ds datastore.DataStore, id string, numElements uint32) (*Handle, error) {
	h := &Handle{
		app:        app,
		id:         id,
		store:      ds,
		bits:       numElements,
		unselected: numElements,
		head: &sequence{
			block: 0x0,
			count: getNumBlocks(numElements),
		},
	}

	if h.store == nil {
		return h, nil
	}

	// Register for status changes
	h.watchForChanges()

	// Get the initial status from the ds if present.
	if err := h.store.GetObject(datastore.Key(h.Key()...), h); err != nil && err != datastore.ErrKeyNotFound {
		return nil, err
	}

	// If the handle is not in store, write it.
	if !h.Exists() {
		if err := h.writeToStore(); err != nil {
			return nil, fmt.Errorf("failed to write bitsequence to store: %v", err)
		}
	}

	return h, nil
}

func (n *network) obtainVxlanID(s *subnet) error {
	//return if the subnet already has a vxlan id assigned
	if s.vni != 0 {
		return nil
	}

	if n.driver.store == nil {
		return fmt.Errorf("no datastore configured. cannot obtain vxlan id")
	}

	for {
		if err := n.driver.store.GetObject(datastore.Key(n.Key()...), n); err != nil {
			return fmt.Errorf("getting network %q from datastore failed %v", n.id, err)
		}

		if s.vni == 0 {
			vxlanID, err := n.driver.vxlanIdm.GetID()
			if err != nil {
				return fmt.Errorf("failed to allocate vxlan id: %v", err)
			}

			n.setVxlanID(s, uint32(vxlanID))
			if err := n.writeToStore(); err != nil {
				n.driver.vxlanIdm.Release(uint64(n.vxlanID(s)))
				n.setVxlanID(s, 0)
				if err == datastore.ErrKeyModified {
					continue
				}
				return fmt.Errorf("network %q failed to update data store: %v", n.id, err)
			}
			return nil
		}
		return nil
	}
}

func (n *network) getEndpointsFromStore() ([]*endpoint, error) {
	var epl []*endpoint

	tmp := endpoint{network: n}
	for _, store := range n.getController().getStores() {
		kvol, err := store.List(datastore.Key(tmp.KeyPrefix()...), &endpoint{network: n})
		if err != nil && err != datastore.ErrKeyNotFound {
			return nil,
				fmt.Errorf("failed to get endpoints for network %s scope %s: %v",
					n.Name(), store.Scope(), err)
		}

		// Continue searching in the next store if no keys found in this store
		if err == datastore.ErrKeyNotFound {
			continue
		}

		for _, kvo := range kvol {
			ep := kvo.(*endpoint)
			ep.network = n
			epl = append(epl, ep)
		}
	}

	return epl, nil
}

func (c *controller) getEndpointFromStore(eid types.UUID) (*endpoint, error) {
	ep := endpoint{id: eid}
	if err := c.store.GetObject(datastore.Key(ep.Key()...), &ep); err != nil {
		return nil, err
	}
	return &ep, nil
}

func (c *controller) getNetworksFromStore() ([]*network, error) {
	var nl []*network

	for _, store := range c.getStores() {
		kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
			&network{ctrlr: c})
		if err != nil && err != datastore.ErrKeyNotFound {
			return nil, fmt.Errorf("failed to get networks for scope %s: %v",
				store.Scope(), err)
		}

		// Continue searching in the next store if no keys found in this store
		if err == datastore.ErrKeyNotFound {
			continue
		}

		for _, kvo := range kvol {
			n := kvo.(*network)
			n.ctrlr = c
			nl = append(nl, n)
		}
	}

	return nl, nil
}

func (c *controller) getNetworkFromStore(nid types.UUID) (*network, error) {
	n := network{id: nid}
	if err := c.store.GetObject(datastore.Key(n.Key()...), &n); err != nil {
		return nil, err
	}
	return &n, nil
}

func (h *Handle) watchForChanges() error {
	h.Lock()
	store := h.store
	h.Unlock()

	if store == nil {
		return nil
	}

	kvpChan, err := store.KVStore().Watch(datastore.Key(h.Key()...), nil)
	if err != nil {
		return err
	}
	go func() {
		for {
			select {
			case kvPair := <-kvpChan:
				// Only process remote update
				if kvPair != nil && (kvPair.LastIndex != h.Index()) {
					err := h.fromDsValue(kvPair.Value)
					if err != nil {
						log.Warnf("Failed to reconstruct bitseq handle from ds watch: %s", err.Error())
					} else {
						h.SetIndex(kvPair.LastIndex)
					}
				}
			}
		}
	}()
	return nil
}

func (n *network) obtainVxlanID() error {
	if n.driver.store == nil {
		return fmt.Errorf("no datastore configured. cannot obtain vxlan id")
	}

	for {
		var vxlanID uint32
		if err := n.driver.store.GetObject(datastore.Key(n.Key()...), n); err != nil {
			if err == datastore.ErrKeyNotFound {
				vxlanID, err = n.driver.vxlanIdm.GetID()
				if err != nil {
					return fmt.Errorf("failed to allocate vxlan id: %v", err)
				}

				n.setVxlanID(vxlanID)
				if err := n.writeToStore(); err != nil {
					n.driver.vxlanIdm.Release(n.vxlanID())
					n.setVxlanID(0)
					if err == datastore.ErrKeyModified {
						continue
					}
					return fmt.Errorf("failed to update data store with vxlan id: %v", err)
				}
				return nil
			}
			return fmt.Errorf("failed to obtain vxlan id from data store: %v", err)
		}
		return nil
	}
}

// NewHandle returns a thread-safe instance of the bitmask handler
func NewHandle(app string, ds datastore.DataStore, id string, numElements uint32) (*Handle, error) {
	h := &Handle{
		app:        app,
		id:         id,
		store:      ds,
		bits:       numElements,
		unselected: numElements,
		head: &Sequence{
			Block: 0x0,
			Count: getNumBlocks(numElements),
		},
	}

	if h.store == nil {
		return h, nil
	}

	// Register for status changes
	h.watchForChanges()

	// Get the initial status from the ds if present.
	if err := h.store.GetObject(datastore.Key(h.Key()...), h); err != nil && err != datastore.ErrKeyNotFound {
		return nil, err
	}

	return h, nil
}

func (c *controller) watchNewNetworks() {
	c.Lock()
	cs := c.store
	c.Unlock()

	cs.KVStore().WatchRange(datastore.Key(datastore.NetworkKeyPrefix), "", 0, func(kvi []store.KVEntry) {
		for _, kve := range kvi {
			var n network
			err := json.Unmarshal(kve.Value(), &n)
			if err != nil {
				log.Error(err)
				continue
			}
			n.dbIndex = kve.LastIndex()
			c.Lock()
			existing, ok := c.networks[n.id]
			c.Unlock()
			if ok && existing.dbIndex == n.dbIndex {
				// Skip any watch notification for a network that has not changed
				continue
			} else if ok {
				// Received an update for an existing network object
				log.Debugf("Skipping network update for %s (%s)", n.name, n.id)
				continue
			}

			c.newNetworkFromStore(&n)
		}
	})
}

func testLocalBackend(t *testing.T, provider, url string, storeConfig *store.Config) {
	netOptions := []config.Option{}
	netOptions = append(netOptions, config.OptionLocalKVProvider(provider))
	netOptions = append(netOptions, config.OptionLocalKVProviderURL(url))
	netOptions = append(netOptions, config.OptionLocalKVProviderConfig(storeConfig))

	ctrl, err := New(netOptions...)
	if err != nil {
		t.Fatalf("Error new controller: %v", err)
	}
	if err := ctrl.ConfigureNetworkDriver("host", options.NewGeneric()); err != nil {
		t.Fatalf("Error initializing host driver: %v", err)
	}
	nw, err := ctrl.NewNetwork("host", "host")
	if err != nil {
		t.Fatalf("Error creating default \"host\" network: %v", err)
	}
	ep, err := nw.CreateEndpoint("newendpoint", []EndpointOption{}...)
	if err != nil {
		t.Fatalf("Error creating endpoint: %v", err)
	}
	store := ctrl.(*controller).localStore.KVStore()
	if exists, err := store.Exists(datastore.Key(datastore.NetworkKeyPrefix, string(nw.ID()))); !exists || err != nil {
		t.Fatalf("Network key should have been created.")
	}
	if exists, err := store.Exists(datastore.Key([]string{datastore.EndpointKeyPrefix, string(nw.ID()), string(ep.ID())}...)); !exists || err != nil {
		t.Fatalf("Endpoint key should have been created.")
	}
	store.Close()

	// test restore of local store
	ctrl, err = New(netOptions...)
	if nw, err = ctrl.NetworkByID(nw.ID()); err != nil {
		t.Fatalf("Error get network %v", err)
	}
	if ep, err = nw.EndpointByID(ep.ID()); err != nil {
		t.Fatalf("Error get endpoint %v", err)
	}
	if err := ep.Delete(); err != nil {
		t.Fatalf("Error delete endpoint %v", err)
	}
	store = ctrl.(*controller).localStore.KVStore()
	if exists, err := store.Exists(datastore.Key([]string{datastore.EndpointKeyPrefix, string(nw.ID()), string(ep.ID())}...)); exists || err != nil {
		t.Fatalf("Endpoint key should have been deleted. ")
	}
}

func (ep *endpoint) networkIDFromKey(key string) (string, error) {
	// endpoint Key structure : docker/libnetwork/endpoint/${network-id}/${endpoint-id}
	// it's an invalid key if the key doesn't have all the 5 key elements above
	keyElements := strings.Split(key, "/")
	if !strings.HasPrefix(key, datastore.Key(datastore.EndpointKeyPrefix)) || len(keyElements) < 5 {
		return "", fmt.Errorf("invalid endpoint key : %v", key)
	}
	// network-id is placed at index=3. pls refer to endpoint.Key() method
	return strings.Split(key, "/")[3], nil
}