func (blockchain *blockchain) persistRawBlock(block *protos.Block, blockNumber uint64) error {
	blockBytes, blockBytesErr := block.Bytes()
	if blockBytesErr != nil {
		return blockBytesErr
	}
	writeBatch := gorocksdb.NewWriteBatch()
	defer writeBatch.Destroy()
	writeBatch.PutCF(db.GetDBHandle().BlockchainCF, encodeBlockNumberDBKey(blockNumber), blockBytes)

	blockHash, err := block.GetHash()
	if err != nil {
		return err
	}

	// Need to check as we suport out of order blocks in cases such as block/state synchronization. This is
	// really blockchain height, not size.
	if blockchain.getSize() < blockNumber+1 {
		sizeBytes := encodeUint64(blockNumber + 1)
		writeBatch.PutCF(db.GetDBHandle().BlockchainCF, blockCountKey, sizeBytes)
		blockchain.size = blockNumber + 1
		blockchain.previousBlockHash = blockHash
	}

	if blockchain.indexer.isSynchronous() {
		blockchain.indexer.createIndexesSync(block, blockNumber, blockHash, writeBatch)
	}

	opt := gorocksdb.NewDefaultWriteOptions()
	defer opt.Destroy()
	err = db.GetDBHandle().DB.Write(opt, writeBatch)
	if err != nil {
		return err
	}
	return nil
}

func (blockchain *blockchain) addPersistenceChangesForNewBlock(ctx context.Context,
	block *protos.Block, stateHash []byte, writeBatch *gorocksdb.WriteBatch) (uint64, error) {
	block = blockchain.buildBlock(block, stateHash)
	if block.NonHashData == nil {
		block.NonHashData = &protos.NonHashData{LocalLedgerCommitTimestamp: util.CreateUtcTimestamp()}
	} else {
		block.NonHashData.LocalLedgerCommitTimestamp = util.CreateUtcTimestamp()
	}
	blockNumber := blockchain.size
	blockHash, err := block.GetHash()
	if err != nil {
		return 0, err
	}
	blockBytes, blockBytesErr := block.Bytes()
	if blockBytesErr != nil {
		return 0, blockBytesErr
	}
	writeBatch.PutCF(db.GetDBHandle().BlockchainCF, encodeBlockNumberDBKey(blockNumber), blockBytes)
	writeBatch.PutCF(db.GetDBHandle().BlockchainCF, blockCountKey, encodeUint64(blockNumber+1))
	if blockchain.indexer.isSynchronous() {
		blockchain.indexer.createIndexesSync(block, blockNumber, blockHash, writeBatch)
	}
	blockchain.lastProcessedBlock = &lastProcessedBlock{block, blockNumber, blockHash}
	return blockNumber, nil
}

func fetchDataNodesFromDBFor(bucketKey *bucketKey) (dataNodes, error) {
	logger.Debug("Fetching from DB data nodes for bucket [%s]", bucketKey)
	openchainDB := db.GetDBHandle()
	itr := openchainDB.GetStateCFIterator()
	defer itr.Close()
	minimumDataKeyBytes := minimumPossibleDataKeyBytesFor(bucketKey)

	var dataNodes dataNodes

	itr.Seek(minimumDataKeyBytes)

	for ; itr.Valid(); itr.Next() {

		// making a copy of key-value bytes because, underlying key bytes are reused by itr.
		// no need to free slices as iterator frees memory when closed.
		keyBytes := statemgmt.Copy(itr.Key().Data())
		valueBytes := statemgmt.Copy(itr.Value().Data())

		dataKey := newDataKeyFromEncodedBytes(keyBytes)
		logger.Debug("Retrieved data key [%s] from DB for bucket [%s]", dataKey, bucketKey)
		if !dataKey.getBucketKey().equals(bucketKey) {
			logger.Debug("Data key [%s] from DB does not belong to bucket = [%s]. Stopping further iteration and returning results [%v]", dataKey, bucketKey, dataNodes)
			return dataNodes, nil
		}
		dataNode := unmarshalDataNode(dataKey, valueBytes)

		logger.Debug("Data node [%s] from DB belongs to bucket = [%s]. Including the key in results...", dataNode, bucketKey)
		dataNodes = append(dataNodes, dataNode)
	}
	logger.Debug("Returning results [%v]", dataNodes)
	return dataNodes, nil
}

// AddChangesForPersistence commits current changes to the database
func (stateTrie *StateTrie) AddChangesForPersistence(writeBatch *gorocksdb.WriteBatch) error {
	if stateTrie.recomputeCryptoHash {
		_, err := stateTrie.ComputeCryptoHash()
		if err != nil {
			return err
		}
	}

	if stateTrie.trieDelta == nil {
		stateTrieLogger.Info("trieDelta is nil. Not writing anything to DB")
		return nil
	}

	openchainDB := db.GetDBHandle()
	lowestLevel := stateTrie.trieDelta.getLowestLevel()
	for level := lowestLevel; level >= 0; level-- {
		changedNodes := stateTrie.trieDelta.deltaMap[level]
		for _, changedNode := range changedNodes {
			if changedNode.markedForDeletion {
				writeBatch.DeleteCF(openchainDB.StateCF, changedNode.trieKey.getEncodedBytes())
				continue
			}
			serializedContent, err := changedNode.marshal()
			if err != nil {
				return err
			}
			writeBatch.PutCF(openchainDB.StateCF, changedNode.trieKey.getEncodedBytes(), serializedContent)
		}
	}
	stateTrieLogger.Debug("Added changes to DB")
	return nil
}

func TestDBStatsOversizedKV(t *testing.T) {
	dbTestWrapper := db.NewTestDBWrapper()
	dbTestWrapper.CleanDB(t)
	defer dbTestWrapper.CloseDB(t)
	defer deleteTestDBDir()

	openchainDB := db.GetDBHandle()
	writeBatch := gorocksdb.NewWriteBatch()
	writeBatch.PutCF(openchainDB.BlockchainCF, []byte("key1"), []byte("value1"))
	writeBatch.PutCF(openchainDB.BlockchainCF, []byte("key2"), generateOversizedValue(0))
	writeBatch.PutCF(openchainDB.BlockchainCF, []byte("key3"), generateOversizedValue(100))
	writeBatch.PutCF(openchainDB.BlockchainCF, []byte("key4"), []byte("value4"))
	dbTestWrapper.WriteToDB(t, writeBatch)

	totalKVs, numOverSizedKVs := scan(openchainDB, "blockchainCF", openchainDB.BlockchainCF, testDetailPrinter)

	if totalKVs != 4 {
		t.Fatalf("totalKVs is not correct. Expected [%d], found [%d]", 4, totalKVs)
	}

	if numOverSizedKVs != 2 {
		t.Fatalf("numOverSizedKVs is not correct. Expected [%d], found [%d]", 2, numOverSizedKVs)
	}

	if numOversizedKeyValues != 2 {
		t.Fatalf("numOversizedKeyValues is not correct. Expected [%d], found [%d]", 2, numOversizedKeyValues)
	}
}

// Functions for persisting and retrieving index data
func addIndexDataForPersistence(block *protos.Block, blockNumber uint64, blockHash []byte, writeBatch *gorocksdb.WriteBatch) error {
	openchainDB := db.GetDBHandle()
	cf := openchainDB.IndexesCF

	// add blockhash -> blockNumber
	indexLogger.Debugf("Indexing block number [%d] by hash = [%x]", blockNumber, blockHash)
	writeBatch.PutCF(cf, encodeBlockHashKey(blockHash), encodeBlockNumber(blockNumber))

	addressToTxIndexesMap := make(map[string][]uint64)
	addressToChaincodeIDsMap := make(map[string][]*protos.ChaincodeID)

	transactions := block.GetTransactions()
	for txIndex, tx := range transactions {
		// add TxID -> (blockNumber,indexWithinBlock)
		writeBatch.PutCF(cf, encodeTxIDKey(tx.Txid), encodeBlockNumTxIndex(blockNumber, uint64(txIndex)))

		txExecutingAddress := getTxExecutingAddress(tx)
		addressToTxIndexesMap[txExecutingAddress] = append(addressToTxIndexesMap[txExecutingAddress], uint64(txIndex))

		switch tx.Type {
		case protos.Transaction_CHAINCODE_DEPLOY, protos.Transaction_CHAINCODE_INVOKE:
			authroizedAddresses, chaincodeID := getAuthorisedAddresses(tx)
			for _, authroizedAddress := range authroizedAddresses {
				addressToChaincodeIDsMap[authroizedAddress] = append(addressToChaincodeIDsMap[authroizedAddress], chaincodeID)
			}
		}
	}
	for address, txsIndexes := range addressToTxIndexesMap {
		writeBatch.PutCF(cf, encodeAddressBlockNumCompositeKey(address, blockNumber), encodeListTxIndexes(txsIndexes))
	}
	return nil
}

func newStateSnapshotIterator(snapshot *gorocksdb.Snapshot) (*StateSnapshotIterator, error) {
	dbItr := db.GetDBHandle().GetStateCFSnapshotIterator(snapshot)
	dbItr.SeekToFirst()
	// skip the root key, because, the value test in Next method is misleading for root key as the value field
	dbItr.Next()
	return &StateSnapshotIterator{dbItr, nil, nil}, nil
}

func (cache *bucketCache) loadAllBucketNodesFromDB() {
	if !cache.isEnabled {
		return
	}
	openchainDB := db.GetDBHandle()
	itr := openchainDB.GetStateCFIterator()
	defer itr.Close()
	itr.Seek([]byte{byte(0)})
	count := 0
	cache.lock.Lock()
	defer cache.lock.Unlock()
	for ; itr.Valid(); itr.Next() {
		key := itr.Key().Data()
		if key[0] != byte(0) {
			itr.Key().Free()
			itr.Value().Free()
			break
		}
		bKey := decodeBucketKey(statemgmt.Copy(itr.Key().Data()))
		nodeBytes := statemgmt.Copy(itr.Value().Data())
		bucketNode := unmarshalBucketNode(&bKey, nodeBytes)
		size := bKey.size() + bucketNode.size()
		cache.size += size
		if cache.size >= cache.maxSize {
			cache.size -= size
			break
		}
		cache.c[bKey] = bucketNode
		itr.Key().Free()
		itr.Value().Free()
		count++
	}
	logger.Info("Loaded buckets data in cache. Total buckets in DB = [%d]. Total cache size:=%d", count, cache.size)
}

func TestStateSnapshotIterator(t *testing.T) {
	testDBWrapper.CreateFreshDB(t)
	stateTrieTestWrapper := newStateTrieTestWrapper(t)
	stateTrie := stateTrieTestWrapper.stateTrie
	stateDelta := statemgmt.NewStateDelta()

	// insert keys
	stateDelta.Set("chaincodeID1", "key1", []byte("value1"), nil)
	stateDelta.Set("chaincodeID2", "key2", []byte("value2"), nil)
	stateDelta.Set("chaincodeID3", "key3", []byte("value3"), nil)
	stateDelta.Set("chaincodeID4", "key4", []byte("value4"), nil)
	stateDelta.Set("chaincodeID5", "key5", []byte("value5"), nil)
	stateDelta.Set("chaincodeID6", "key6", []byte("value6"), nil)
	stateTrie.PrepareWorkingSet(stateDelta)
	stateTrieTestWrapper.PersistChangesAndResetInMemoryChanges()
	//check that the key is persisted
	testutil.AssertEquals(t, stateTrieTestWrapper.Get("chaincodeID1", "key1"), []byte("value1"))
	testutil.AssertEquals(t, stateTrieTestWrapper.Get("chaincodeID2", "key2"), []byte("value2"))
	testutil.AssertEquals(t, stateTrieTestWrapper.Get("chaincodeID3", "key3"), []byte("value3"))
	testutil.AssertEquals(t, stateTrieTestWrapper.Get("chaincodeID4", "key4"), []byte("value4"))
	testutil.AssertEquals(t, stateTrieTestWrapper.Get("chaincodeID5", "key5"), []byte("value5"))
	testutil.AssertEquals(t, stateTrieTestWrapper.Get("chaincodeID6", "key6"), []byte("value6"))

	// take db snapeshot
	dbSnapshot := db.GetDBHandle().GetSnapshot()

	stateDelta1 := statemgmt.NewStateDelta()
	// delete a few keys
	stateDelta1.Delete("chaincodeID1", "key1", nil)
	stateDelta1.Delete("chaincodeID3", "key3", nil)
	stateDelta1.Delete("chaincodeID4", "key4", nil)
	stateDelta1.Delete("chaincodeID6", "key6", nil)

	// update remaining keys
	stateDelta1.Set("chaincodeID2", "key2", []byte("value2_new"), nil)
	stateDelta1.Set("chaincodeID5", "key5", []byte("value5_new"), nil)

	stateTrie.PrepareWorkingSet(stateDelta1)
	stateTrieTestWrapper.PersistChangesAndResetInMemoryChanges()
	//check that the keys are updated
	testutil.AssertNil(t, stateTrieTestWrapper.Get("chaincodeID1", "key1"))
	testutil.AssertNil(t, stateTrieTestWrapper.Get("chaincodeID3", "key3"))
	testutil.AssertNil(t, stateTrieTestWrapper.Get("chaincodeID4", "key4"))
	testutil.AssertNil(t, stateTrieTestWrapper.Get("chaincodeID6", "key6"))
	testutil.AssertEquals(t, stateTrieTestWrapper.Get("chaincodeID2", "key2"), []byte("value2_new"))
	testutil.AssertEquals(t, stateTrieTestWrapper.Get("chaincodeID5", "key5"), []byte("value5_new"))

	itr, err := newStateSnapshotIterator(dbSnapshot)
	testutil.AssertNoError(t, err, "Error while getting state snapeshot iterator")

	stateDeltaFromSnapshot := statemgmt.NewStateDelta()
	for itr.Next() {
		keyBytes, valueBytes := itr.GetRawKeyValue()
		t.Logf("key=[%s], value=[%s]", string(keyBytes), string(valueBytes))
		chaincodeID, key := statemgmt.DecodeCompositeKey(keyBytes)
		stateDeltaFromSnapshot.Set(chaincodeID, key, valueBytes, nil)
	}
	testutil.AssertEquals(t, stateDelta, stateDeltaFromSnapshot)
}

// DeleteState deletes ALL state keys/values from the DB. This is generally
// only used during state synchronization when creating a new state from
// a snapshot.
func (state *State) DeleteState() error {
	state.ClearInMemoryChanges(false)
	err := db.GetDBHandle().DeleteState()
	if err != nil {
		logger.Errorf("Error deleting state: %s", err)
	}
	return err
}

func fetchBlockNumberByBlockHashFromDB(blockHash []byte) (uint64, error) {
	blockNumberBytes, err := db.GetDBHandle().GetFromIndexesCF(encodeBlockHashKey(blockHash))
	if err != nil {
		return 0, err
	}
	blockNumber := decodeBlockNumber(blockNumberBytes)
	return blockNumber, nil
}

func fetchTransactionIndexByIDFromDB(txID string) (uint64, uint64, error) {
	blockNumTxIndexBytes, err := db.GetDBHandle().GetFromIndexesCF(encodeTxIDKey(txID))
	if err != nil {
		return 0, 0, err
	}
	if blockNumTxIndexBytes == nil {
		return 0, 0, ErrResourceNotFound
	}
	return decodeBlockNumTxIndex(blockNumTxIndexBytes)
}

func fetchBlockFromDB(blockNumber uint64) (*protos.Block, error) {
	blockBytes, err := db.GetDBHandle().GetFromBlockchainCF(encodeBlockNumberDBKey(blockNumber))
	if err != nil {
		return nil, err
	}
	if blockBytes == nil {
		return nil, nil
	}
	return protos.UnmarshallBlock(blockBytes)
}

func fetchBlockchainSizeFromDB() (uint64, error) {
	bytes, err := db.GetDBHandle().GetFromBlockchainCF(blockCountKey)
	if err != nil {
		return 0, err
	}
	if bytes == nil {
		return 0, nil
	}
	return decodeToUint64(bytes), nil
}

func newRangeScanIterator(chaincodeID string, startKey string, endKey string) (*RangeScanIterator, error) {
	dbItr := db.GetDBHandle().GetStateCFIterator()
	itr := &RangeScanIterator{
		dbItr:       dbItr,
		chaincodeID: chaincodeID,
		startKey:    startKey,
		endKey:      endKey,
	}
	itr.seekForStartKeyWithinBucket(1)
	return itr, nil
}

func fetchDataNodeFromDB(dataKey *dataKey) (*dataNode, error) {
	openchainDB := db.GetDBHandle()
	nodeBytes, err := openchainDB.GetFromStateCF(dataKey.getEncodedBytes())
	if err != nil {
		return nil, err
	}
	if util.IsNil(nodeBytes) {
		return nil, nil
	}
	return unmarshalDataNode(dataKey, nodeBytes), nil
}

func fetchBlockchainSizeFromSnapshot(snapshot *gorocksdb.Snapshot) (uint64, error) {
	blockNumberBytes, err := db.GetDBHandle().GetFromBlockchainCFSnapshot(snapshot, blockCountKey)
	if err != nil {
		return 0, err
	}
	var blockNumber uint64
	if blockNumberBytes != nil {
		blockNumber = decodeToUint64(blockNumberBytes)
	}
	return blockNumber, nil
}

func fetchBucketNodeFromDB(bucketKey *bucketKey) (*bucketNode, error) {
	openchainDB := db.GetDBHandle()
	nodeBytes, err := openchainDB.GetFromStateCF(bucketKey.getEncodedBytes())
	if err != nil {
		return nil, err
	}
	if nodeBytes == nil {
		return nil, nil
	}
	return unmarshalBucketNode(bucketKey, nodeBytes), nil
}

// FetchStateDeltaFromDB fetches the StateDelta corrsponding to given blockNumber
func (state *State) FetchStateDeltaFromDB(blockNumber uint64) (*statemgmt.StateDelta, error) {
	stateDeltaBytes, err := db.GetDBHandle().GetFromStateDeltaCF(encodeStateDeltaKey(blockNumber))
	if err != nil {
		return nil, err
	}
	if stateDeltaBytes == nil {
		return nil, nil
	}
	stateDelta := statemgmt.NewStateDelta()
	stateDelta.Unmarshal(stateDeltaBytes)
	return stateDelta, nil
}

func fetchLastIndexedBlockNumFromDB() (zerothBlockIndexed bool, lastIndexedBlockNum uint64, err error) {
	lastIndexedBlockNumberBytes, err := db.GetDBHandle().GetFromIndexesCF(lastIndexedBlockKey)
	if err != nil {
		return
	}
	if lastIndexedBlockNumberBytes == nil {
		return
	}
	lastIndexedBlockNum = decodeBlockNumber(lastIndexedBlockNumberBytes)
	zerothBlockIndexed = true
	return
}