// SETUP AND TEARDOWN ///////////////////////////////////////////////
func (s *XLSuite) setUpHashTest() {
	var err error
	found, err := xf.PathExists(dataPath)
	if !found {
		// MODE SUSPECT
		if err = os.MkdirAll(dataPath, 0775); err != nil {
			fmt.Printf("error creating %s: %v\n", dataPath, err)
		}
	}
	found, err = xf.PathExists(uPath)
	if !found {
		// MODE SUSPECT
		if err = os.MkdirAll(uPath, 0775); err != nil {
			fmt.Printf("error creating %s: %v\n", uPath, err)
		}
	}
	found, err = xf.PathExists(uInDir)
	if !found {
		// MODE SUSPECT
		if err = os.MkdirAll(uInDir, 0775); err != nil {
			fmt.Printf("error creating %s: %v\n", uInDir, err)
		}
	}
	found, err = xf.PathExists(uTmpDir)
	if !found {

		// MODE SUSPECT
		if err = os.MkdirAll(uTmpDir, 0775); err != nil {
			fmt.Printf("error creating %s: %v\n", uTmpDir, err)
		}
	}
}

func (u *U16x16) PutData2(data []byte, key string) (length int64, hash string, err error) {
	s := sha256.New()
	s.Write(data)
	hash = hex.EncodeToString(s.Sum(nil))
	if hash != key {
		fmt.Printf("expected data to have key %s, but content key is %s",
			key, hash)
		err = errors.New("content/key mismatch")
		return
	}
	length = int64(len(data))
	topSubDir := hash[0:1]
	lowerDir := hash[1:2]
	targetDir := filepath.Join(u.path, topSubDir, lowerDir)
	found, err := xf.PathExists(targetDir)
	if err == nil && !found {
		err = os.MkdirAll(targetDir, 0775)
	}
	fullishPath := filepath.Join(targetDir, key[2:])
	found, err = xf.PathExists(fullishPath)
	if !found {
		var dest *os.File
		dest, err = os.Create(fullishPath)
		if err == nil {
			var count int
			defer dest.Close()
			count, err = dest.Write(data)
			if err == nil {
				length = int64(count)
			}
		}
	}
	return
}

func (s *XLSuite) doTestLoadEntries(c *C, rng *xr.PRNG, whichSHA int) {
	K := 16 + rng.Intn(16)

	// create a unique name for a scratch file
	pathToFile := filepath.Join("tmp", rng.NextFileName(16))
	found, err := xf.PathExists(pathToFile)
	c.Assert(err, IsNil)
	for found {
		pathToFile = filepath.Join("tmp", rng.NextFileName(16))
		found, err = xf.PathExists(pathToFile)
		c.Assert(err, IsNil)
	}
	f, err := os.OpenFile(pathToFile, os.O_CREATE|os.O_WRONLY, 0600)
	c.Assert(err, IsNil)

	// create K entries, saving them in a slice while writing them
	// to disk
	var entries []*LogEntry
	for i := 0; i < K; i++ {
		t, key, nodeID, src, path := s.makeEntryData(c, rng, whichSHA)
		entry, err := NewLogEntry(t, key, nodeID, src, path)
		c.Assert(err, IsNil)
		strEntry := entry.String()
		entries = append(entries, entry)
		var count int
		count, err = f.WriteString(strEntry + "\n")
		c.Assert(err, IsNil)
		c.Assert(count, Equals, len(strEntry)+1)
	}
	f.Close()
	c.Assert(len(entries), Equals, K)

	// use UpaxServer.LoadEntries to load the stuff in the file.
	m, err := xi.NewNewIDMap()
	c.Assert(err, IsNil)
	count, err := loadEntries(pathToFile, m, whichSHA)
	c.Assert(err, IsNil)
	c.Assert(count, Equals, K) // K entries loaded.

	for i := 0; i < K; i++ {
		var entry, eInMap *LogEntry
		var whatever interface{}
		entry = entries[i]
		key := entry.key
		whatever, err = m.Find(key)
		c.Assert(err, IsNil)
		c.Assert(whatever, NotNil)
		eInMap = whatever.(*LogEntry)

		// DEBUG
		// XXX NEED LogEntry.Equal()
		// END

		c.Assert(bytes.Equal(key, eInMap.key), Equals, true)
	}
}

func (s *XLSuite) doTestExists(c *C, u UI, digest hash.Hash) {
	//we are testing whether = u.Exists( key) and whether = u.KeyExists()

	rng := u.GetRNG()
	dLen, dPath := rng.NextDataFile(dataPath, 16*1024, 1)
	var dKey string
	var err error
	switch whichSHA {
	case xu.USING_SHA1:
		dKey, err = FileHexSHA1(dPath)
	case xu.USING_SHA2:
		dKey, err = FileHexSHA2(dPath)
	case xu.USING_SHA3:
		dKey, err = FileHexSHA3(dPath)
		// XXX DEFAULT = ERROR
	}
	c.Assert(err, Equals, nil)
	var uLen int64
	var uKey string
	switch whichSHA {
	case xu.USING_SHA1:
		uLen, uKey, err = u.CopyAndPut1(dPath, dKey)
	case xu.USING_SHA2:
		uLen, uKey, err = u.CopyAndPut2(dPath, dKey)
	case xu.USING_SHA3:
		uLen, uKey, err = u.CopyAndPut3(dPath, dKey)
		// XXX DEFAULT = ERROR
	}
	c.Assert(err, Equals, nil)
	c.Assert(dLen, Equals, uLen)
	kPath, err := u.GetPathForHexKey(uKey)
	c.Assert(err, Equals, nil)
	found, err := xf.PathExists(kPath)
	c.Assert(err, IsNil)
	c.Assert(found, Equals, true)

	bKey, err := hex.DecodeString(uKey)
	c.Assert(err, IsNil)

	found, err = u.HexKeyExists(uKey) // string version of key
	c.Assert(err, IsNil)
	c.Assert(found, Equals, true)
	found, err = u.ByteKeyExists(bKey) // binary version of key
	c.Assert(err, IsNil)
	c.Assert(found, Equals, true)

	os.Remove(kPath)

	found, err = xf.PathExists(kPath) // string version
	c.Assert(err, IsNil)
	c.Assert(found, Equals, false)

	found, err = u.ByteKeyExists(bKey) // binary version of key
	c.Assert(err, IsNil)
	c.Assert(found, Equals, false)
}

// UNIT TESTS ///////////////////////////////////////////////////////
func (s *XLSuite) doTestCopyAndPut(
	c *C, u UI, digest hash.Hash) {
	//we are testing uLen, uKey, err = u.CopyAndPut3(path, key)

	// create a random file
	rng := u.GetRNG()
	dLen, dPath := rng.NextDataFile(dataPath, 16*1024, 1) //  maxLen, minLen
	var dKey string
	var err error
	switch whichSHA {
	case xu.USING_SHA1:
		dKey, err = FileHexSHA1(dPath)
	case xu.USING_SHA2:
		dKey, err = FileHexSHA2(dPath)
	case xu.USING_SHA3:
		dKey, err = FileHexSHA3(dPath)
		// XXX DEFAULT = ERROR
	}
	c.Assert(err, Equals, nil) // actual, Equals, expected

	// invoke function
	var uLen int64
	var uKey string
	switch whichSHA {
	case xu.USING_SHA1:
		uLen, uKey, err = u.CopyAndPut1(dPath, dKey)
	case xu.USING_SHA2:
		uLen, uKey, err = u.CopyAndPut2(dPath, dKey)
	case xu.USING_SHA3:
		uLen, uKey, err = u.CopyAndPut3(dPath, dKey)
		// XXX DEFAULT = ERROR
	}
	c.Assert(err, Equals, nil)
	c.Assert(dLen, Equals, uLen)
	c.Assert(dKey, Equals, uKey)

	// verify that original and copy both exist
	found, err := xf.PathExists(dPath)
	c.Assert(err, IsNil)
	c.Assert(found, Equals, true)
	xPath, err := u.GetPathForHexKey(uKey)
	c.Assert(err, IsNil)
	found, err = xf.PathExists(xPath)
	c.Assert(err, IsNil)
	c.Assert(found, Equals, true)

	// HACK - SIMPLEST Keccak TEST VECTOR
	if whichSHA == xu.USING_SHA3 {
		dKey, err = FileHexSHA3("abc")
		fmt.Printf("SHA3-256 for 'abc' is %s\n", dKey)
	}
	// END HACK
}

// These are operations on the file system.  Directory depth is at least 1
// and no more than 'depth'.  Likewise for width, the number of
// files in a directory, where a file is either a data file or a subdirectory.
// The number of bytes in a file is at least minLen and less than maxLen.
// Subdirectory names may be random
//
// XXX Changed to return an int64 length.
//
func (p *PRNG) NextDataFile(dirName string, maxLen int, minLen int) (
	int64, string) {

	// silently convert parameters to reasonable values
	if minLen < 0 {
		minLen = 0
	}
	if maxLen < minLen+1 {
		maxLen = minLen + 1
	}

	// create the data directory if it does not exist
	dirExists, err := xf.PathExists(dirName)
	if err != nil {
		panic(err)
	}
	if !dirExists {
		os.MkdirAll(dirName, 0755)
	}

	// loop until the file does not exist
	pathToFile := dirName + "/" + p.NextFileName(16)
	pathExists, err := xf.PathExists(pathToFile)
	if err != nil {
		panic(err)
	}
	for pathExists {
		pathToFile := dirName + "/" + p.NextFileName(16)
		pathExists, err = xf.PathExists(pathToFile)
		if err != nil {
			panic(err)
		}
	}
	count := minLen + int(p.NextFloat32()*float32((maxLen-minLen)))
	data := make([]byte, count)
	p.NextBytes(data) // fill with random bytes
	fo, err := os.Create(pathToFile)
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := fo.Close(); err != nil {
			panic(err)
		}
	}() // XXX wakaranai
	// XXX this should be chunked
	// XXX data should be slice
	if _, err := fo.Write(data); err != nil {
		panic(err)
	}
	// XXX respec to also return err
	return int64(count), pathToFile
}

// tmp is the path to a local file which will be renamed into U (or deleted
// if it is already present in U)
// u.path is an absolute or relative path to a U directory organized 16x16
// key is an sha1 content hash.
// If the operation succeeds we return the length of the file (which must
// not be zero.  Otherwise we return 0.
// we don't do much checking.
//
func (u *U16x16) Put1(inFile, key string) (
	length int64, hash string, err error) {

	var (
		found                          bool
		fullishPath                    string
		topSubDir, lowerDir, targetDir string
	)
	hash, err = FileHexSHA1(inFile)
	if err != nil {
		fmt.Printf("DEBUG: FileHexSHA1 returned error %v\n", err)
		return
	}
	if hash != key {
		fmt.Printf("expected %s to have key %s, but the content key is %s\n",
			inFile, key, hash)
		err = errors.New("IllegalArgument: Put1: key does not match content")
		return
	}
	info, err := os.Stat(inFile)
	if err != nil {
		return
	}
	length = info.Size()
	topSubDir = hash[0:1]
	lowerDir = hash[1:2]
	targetDir = filepath.Join(u.path, topSubDir, lowerDir)
	found, err = xf.PathExists(targetDir)
	if err == nil && !found {
		// XXX MODE IS SUSPECT
		err = os.MkdirAll(targetDir, 0775)

	}
	if err == nil {
		fullishPath = filepath.Join(targetDir, key[2:])
		found, err = xf.PathExists(fullishPath)
	}
	if err == nil {
		if found {
			// drop the temporary input file
			err = os.Remove(inFile)
		} else {
			// rename the temporary file into U
			err = os.Rename(inFile, fullishPath)
			if err == nil {
				err = os.Chmod(fullishPath, 0444)
			}
		}
	}
	return
}

func (u2 *U256x256) CopyAndPut2(path, key string) (
	written int64, hash string, err error) {
	// the temporary file MUST be created on the same device
	tmpFileName := filepath.Join(u2.tmpDir, u2.rng.NextFileName(16))
	found, _ := xf.PathExists(tmpFileName) // XXX error ignored
	for found {
		tmpFileName = filepath.Join(u2.tmpDir, u2.rng.NextFileName(16))
		found, _ = xf.PathExists(tmpFileName)
	}
	written, err = CopyFile(tmpFileName, path) // dest <== src
	if err == nil {
		written, hash, err = u2.Put2(tmpFileName, key)
	}
	return
}

func (u *U16x16) ByteKeyExists(key []byte) (found bool, err error) {
	path, err := u.GetPathForByteKey(key)
	if err == nil {
		found, err = xf.PathExists(path)
	}
	return
}

func (s *XLSuite) verifyLeafSHA(c *C, rng *xr.PRNG,
	node MerkleNodeI, pathToFile string, whichSHA int) {

	c.Assert(node.IsLeaf(), Equals, true)
	found, err := xf.PathExists(pathToFile)
	c.Assert(err, IsNil)
	c.Assert(found, Equals, true)
	data, err := ioutil.ReadFile(pathToFile)
	c.Assert(err, IsNil)
	c.Assert(data, NotNil)

	var sha hash.Hash
	switch whichSHA {
	case xu.USING_SHA1:
		sha = sha1.New()
	case xu.USING_SHA2:
		sha = sha256.New()
	case xu.USING_SHA3:
		sha = sha3.New256()
		// XXX DEFAULT = ERROR
	}
	sha.Write(data)
	sum := sha.Sum(nil)
	c.Assert(node.GetHash(), DeepEquals, sum)
}

func (u *U16x16) GetData2(key string) (data []byte, err error) {
	var (
		found bool
		path  string
	)
	path, err = u.GetPathForKey(key)
	if err == nil {
		found, err = xf.PathExists(path)
	}
	if err == nil && !found {
		err = FileNotFound
	}
	if err == nil {
		var src *os.File
		if src, err = os.Open(path); err != nil {
			return
		}
		defer src.Close()
		var count int
		// XXX THIS WILL NOT WORK FOR LARGER FILES!  It will ignore
		//     anything over 128 KB
		data = make([]byte, DEFAULT_BUFFER_SIZE)
		count, err = src.Read(data)
		// XXX COUNT IS IGNORED
		_ = count
	}
	return
}

func (u *U16x16) HexKeyExists(key string) (found bool, err error) {
	path, err := u.GetPathForHexKey(key)
	if err == nil {
		found, err = xf.PathExists(path)
	}
	return
}

// PutData2 ---------------------------------------------------------
func (u *UFlat) PutData2(data []byte, key string) (
	length int64, hash string, err error) {

	var fullishPath string
	var found bool

	s := sha256.New()
	s.Write(data)
	hash = hex.EncodeToString(s.Sum(nil))
	if hash != key {
		fmt.Printf("expected data to have key %s, but content key is %s",
			key, hash)
		err = errors.New("content/key mismatch")
		return
	}
	length = int64(len(data))

	if err == nil {
		fullishPath = filepath.Join(u.path, key)
		found, err = xf.PathExists(fullishPath)
		if err == nil && !found {
			var dest *os.File
			dest, err = os.Create(fullishPath)
			if err == nil {
				var count int
				defer dest.Close()
				count, err = dest.Write(data)
				if err == nil {
					length = int64(count)
				}
			}
		}
	}
	return
}

func (u2 *U256x256) CopyAndPut1(path, key string) (
	written int64, hash string, err error) {
	// the temporary file MUST be created on the same device
	// xxx POSSIBLE RACE CONDITION
	tmpFileName := filepath.Join(u2.tmpDir, u2.rng.NextFileName(16))
	found, err := xf.PathExists(tmpFileName)
	for found {
		tmpFileName = filepath.Join(u2.tmpDir, u2.rng.NextFileName(16))
		found, err = xf.PathExists(tmpFileName)
	}
	written, err = CopyFile(tmpFileName, path) // dest <== src
	if err == nil {
		written, hash, err = u2.Put1(tmpFileName, key)
	}
	return
}

// Copy the file at path to a randomly-named temporary file under U/tmp.
// If that operation succeeds, we then attempt to rename the file into
// the appropriate U data subdirectory.  If the file is already present,
// we silently discard the copy.  Returns the length of the file in bytes,
// its actual content hash, and any error.
//
func (u *UFlat) CopyAndPut3(path, key string) (
	written int64, hash string, err error) {

	// the temporary file MUST be created on the same device
	// xxx POSSIBLE RACE CONDITION
	tmpFileName := filepath.Join(u.tmpDir, u.rng.NextFileName(16))
	found, _ := xf.PathExists(tmpFileName) // XXX error ignored
	for found {
		tmpFileName = filepath.Join(u.tmpDir, u.rng.NextFileName(16))
		found, _ = xf.PathExists(tmpFileName)
	}
	written, err = CopyFile(tmpFileName, path) // dest <== src
	if err == nil {
		written, hash, err = u.Put3(tmpFileName, key)
	}
	return
}

func (u2 *U256x256) PutData1(data []byte, key string) (
	length int64, hash string, err error) {

	var fullishPath string
	var found bool

	s := sha1.New()
	s.Write(data)
	hash = hex.EncodeToString(s.Sum(nil))
	if hash != key {
		fmt.Printf("expected data to have key %s, but content key is %s",
			key, hash)
		err = errors.New("content/key mismatch")
		return
	}
	length = int64(len(data))
	topSubDir := hash[0:2]
	lowerDir := hash[2:4]
	targetDir := filepath.Join(u2.path, topSubDir, lowerDir)
	found, err = xf.PathExists(targetDir)
	if err == nil && !found {
		// MODE QUESTIONABLE
		err = os.MkdirAll(targetDir, 0775)
	}
	if err == nil {
		fullishPath = filepath.Join(targetDir, key[4:])
		found, err = xf.PathExists(fullishPath)
		if err == nil && !found {
			var dest *os.File
			dest, err = os.Create(fullishPath)
			if err == nil {
				var count int
				defer dest.Close()
				count, err = dest.Write(data)
				if err == nil {
					length = int64(count)
				}
			}
		}
	}
	return
}

func setUpMB3(c *C, rng *xr.PRNG) (
	filter *MappedBloomSHA, m, k uint, keys [][]byte, backingFile string) {

	m = 20
	k = 8
	keys = make([][]byte, 100)
	for i := 0; i < 100; i++ {
		keys[i] = make([]byte, 20)
	}
	backingFile = "tmp/" + rng.NextFileName(8)
	// make sure the file does not already exist
	found, err := xf.PathExists(backingFile)
	c.Assert(err, IsNil)
	for found {
		backingFile = "tmp/" + rng.NextFileName(8)
		found, err = xf.PathExists(backingFile)
		c.Assert(err, IsNil)
	}
	return
}

func CreateMerkleDocFromFileSystem(pathToDir string, whichSHA int,
	exclusions, matches []string) (md *MerkleDoc, err error) {

	if len(pathToDir) == 0 {
		err = NilPath
	}
	if err == nil {
		var found bool
		found, err = xf.PathExists(pathToDir)
		if err == nil && !found {
			err = FileNotFound
		}
	}
	// get the path to the directory, excluding the directory name
	var (
		path string
		// dirName		string
		exRE, matchRE *re.Regexp
		tree          *MerkleTree
	)
	if strings.HasSuffix(pathToDir, "/") {
		pathToDir = pathToDir[:len(pathToDir)-1] // drop trailing slash
	}
	parts := strings.Split(pathToDir, "/")
	if len(parts) == 1 {
		path = "."
		// dirName = pathToDir
	} else {
		partCount := len(parts)
		// dirName = parts[partCount - 1]
		parts = parts[:partCount-1]
		path = strings.Join(parts, "/")
	}
	if exclusions != nil {
		exRE, err = MakeExRE(exclusions)
		if err == nil && matches != nil {
			matchRE, err = MakeMatchRE(matches)
		}
	}
	if err == nil {
		tree, err = CreateMerkleTreeFromFileSystem(
			pathToDir, whichSHA, exRE, matchRE)
		if err == nil {
			// "creates the hash"
			md, err = NewMerkleDoc(path, whichSHA, false, tree, exRE, matchRE)
			if err == nil {
				md.bound = true
			}
		}
	}
	return
}

func SHA3File(pathToFile string) (hash []byte, err error) {
	var data []byte
	found, err := xf.PathExists(pathToFile)
	if err == nil && !found {
		err = FileNotFound
	}
	if err == nil {
		data, err = ioutil.ReadFile(pathToFile)
		if err == nil {
			digest := sha3.New256()
			digest.Write(data)
			hash = digest.Sum(nil)
		}
	}
	return
}

// returns the SHA1 binHash of the contents of a file
func FileBinSHA1(path string) (binHash []byte, err error) {
	var data2 []byte
	binHash = xu.SHA1_BIN_NONE
	found, err := xf.PathExists(path)
	if err == nil && !found {
		err = errors.New("IllegalArgument: empty path or non-existent file")
	}
	if err == nil {
		data2, err = ioutil.ReadFile(path)
	}
	if err == nil {
		d2 := sha1.New()
		d2.Write(data2)
		binHash = d2.Sum(nil)
	}
	return
}