func TestImportCopy(t *testing.T) {
	b := []byte(`¡åéîòü!`)

	f, err := ioutil.TempFile("", "temp-test")
	if err != nil {
		t.Fatal(err)
	}
	if _, err := f.Write(b); err != nil {
		t.Fatal(err)
	}
	f.Close()

	d := diskv.New(diskv.Options{
		BasePath: "test-import-copy",
	})
	defer d.EraseAll()

	if err := d.Import(f.Name(), "key", false); err != nil {
		t.Fatal(err)
	}

	if _, err := os.Stat(f.Name()); err != nil {
		t.Errorf("expected temp file to remain, but got err = %v", err)
	}
}

func main() {
	d := diskv.New(diskv.Options{
		BasePath:     "data",
		Transform:    blockTransform,
		CacheSizeMax: 1024 * 1024, // 1MB
	})

	for _, valueStr := range []string{
		"I am the very model of a modern Major-General",
		"I've information vegetable, animal, and mineral",
		"I know the kings of England, and I quote the fights historical",
		"From Marathon to Waterloo, in order categorical",
		"I'm very well acquainted, too, with matters mathematical",
		"I understand equations, both the simple and quadratical",
		"About binomial theorem I'm teeming with a lot o' news",
		"With many cheerful facts about the square of the hypotenuse",
	} {
		d.Write(md5sum(valueStr), []byte(valueStr))
	}

	var keyCount int
	for key := range d.Keys() {
		val, err := d.Read(key)
		if err != nil {
			panic(fmt.Sprintf("key %s had no value", key))
		}
		fmt.Printf("%s: %s\n", key, val)
		keyCount++
	}
	fmt.Printf("%d total keys\n", keyCount)

	// d.EraseAll() // leave it commented out to see how data is kept on disk
}

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Updates can be triggered by sending SIGUSR1.")
		fmt.Fprintln(os.Stderr, "Automatic updates can be toggled by sending SIGUSR2.")
		fmt.Fprintln(os.Stderr, "A cache purge can be triggered by sending SIGHUP.")
		fmt.Fprintln(os.Stderr)
	}
	flag.Parse()
	subreddits = strings.Split(*sr, ",")
	log.Printf("watching subreddits: %v\n", subreddits)
	userBlacklist = strings.Split(*ul, ",")
	log.Printf("ignoring users: %v\n", userBlacklist)
	cacheDir := fmt.Sprintf("%s/dereddit.cache", os.TempDir())
	os.Mkdir(*rssDir, 0777)
	if *apiKey == "" {
		log.Fatalln("api key not specified")
	}
	o := diskv.Options{
		BasePath:    cacheDir,
		Compression: diskv.NewGzipCompression(),
		PathPerm:    0755,
		FilePerm:    0666,
	}
	cache = diskv.New(o)
	log.Printf("cache %s opened\n", cacheDir)
	log.Printf("outputting rss feeds to %s\n", *rssDir)
}

func TestKeysCancel(t *testing.T) {
	d := diskv.New(diskv.Options{
		BasePath: "test-data",
	})
	defer d.EraseAll()

	for k, v := range keysTestData {
		d.Write(k, []byte(v))
	}

	var (
		cancel      = make(chan struct{})
		received    = 0
		cancelAfter = len(keysTestData) / 2
	)

	for key := range d.Keys(cancel) {
		received++

		if received >= cancelAfter {
			close(cancel)
			runtime.Gosched() // allow walker to detect cancel
		}

		t.Logf("received %d: %q", received, key)
	}

	if want, have := cancelAfter, received; want != have {
		t.Errorf("want %d, have %d")
	}
}

func main() {
	// Simplest transform function: put all the data files into the base dir.
	flatTransform := func(s string) []string {
		ss := strings.Split(s, ".")
		return ss[0 : len(ss)-1]
	}

	// Initialize a new diskv store, rooted at "my-data-dir", with a 1MB cache.
	d := diskv.New(diskv.Options{
		BasePath:     "my-data-dir",
		Transform:    flatTransform,
		CacheSizeMax: 1024 * 1024,
		PathPerm:     0750,
		FilePerm:     0640,
	})

	// Write three bytes to the key "alpha".
	key := "alpha"
	d.Write(key, []byte{'1', '2', '3'})
	d.Write("beta", []byte{'4', '5', '6'})
	d.Write("sub.alfa", []byte("Hello alpa date"))
	d.Write("sub.omega", []byte("Good buy omega"))

	// Read the value back out of the store.
	value, _ := d.Read(key)
	fmt.Printf("%v\n", value)

	// Erase the key+value from the store (and the disk).
	d.Erase(key)
}

func main() {
	flag.Parse()

	if *version {
		fmt.Printf("%v\nBuild: %v\n", VERSION, BUILD_DATE)
		return
	}

	var c httpcache.Cache
	if *cacheDir != "" {
		d := diskv.New(diskv.Options{
			BasePath:     *cacheDir,
			CacheSizeMax: *cacheSize * 1024 * 1024,
		})
		c = diskcache.NewWithDiskv(d)
	} else if *cacheSize != 0 {
		c = httpcache.NewMemoryCache()
	}

	p := imageproxy.NewProxy(nil, c)
	if *whitelist != "" {
		p.Whitelist = strings.Split(*whitelist, ",")
	}
	if *referrers != "" {
		p.Referrers = strings.Split(*referrers, ",")
	}
	if *signatureKey != "" {
		key := []byte(*signatureKey)
		if strings.HasPrefix(*signatureKey, "@") {
			file := strings.TrimPrefix(*signatureKey, "@")
			var err error
			key, err = ioutil.ReadFile(file)
			if err != nil {
				log.Fatalf("error reading signature file: %v", err)
			}
		}
		p.SignatureKey = key
	}
	if *baseURL != "" {
		var err error
		p.DefaultBaseURL, err = url.Parse(*baseURL)
		if err != nil {
			log.Fatalf("error parsing baseURL: %v", err)
		}
	}

	p.ScaleUp = *scaleUp

	server := &http.Server{
		Addr:    *addr,
		Handler: p,
	}

	fmt.Printf("imageproxy (version %v) listening on %s\n", VERSION, server.Addr)
	err := server.ListenAndServe()
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

// New returns a new Cache that will store files in basePath
func New(basePath string) *Cache {
	return &Cache{
		d: diskv.New(diskv.Options{
			BasePath:     basePath,
			CacheSizeMax: 100 * 1024 * 1024, // 100MB
		}),
	}
}

func diskCache(path string) *diskcache.Cache {
	d := diskv.New(diskv.Options{
		BasePath: path,

		// For file "c0ffee", store file as "c0/ff/c0ffee"
		Transform: func(s string) []string { return []string{s[0:2], s[2:4]} },
	})
	return diskcache.NewWithDiskv(d)
}

func NewPersistentStorageEngine() StorageEngine {
	storageEngine := new(PersistentStorageEngine)

	// TODO set up a folder structure for < 1k entries per folder
	// actual TODO set up a proper datastore
	flatTransform := func(s string) []string { return []string{} }
	storageEngine.tweetStore = diskv.New(diskv.Options{
		BasePath:     "storage/tweets",
		Transform:    flatTransform,
		CacheSizeMax: 1024 * 1024,
	})
	storageEngine.gameStateStore = diskv.New(diskv.Options{
		BasePath:     "storage/game-states",
		Transform:    flatTransform,
		CacheSizeMax: 1024 * 1024,
	})

	return storageEngine
}

func newInformer(graph, entity string) Informer {
	return &informer{
		d: diskv.New(diskv.Options{
			BasePath:     basePathFor(graph, entity),
			Transform:    decimalSplit(4),
			CacheSizeMax: 0,
			Compression:  diskv.NewZlibCompression(),
		}),
	}
}

func NewConfigMgr() *ConfigMgr {
	path := "/tmp/db-diskv"

	diskv := diskv.New(diskv.Options{
		BasePath:     path,
		Transform:    blockTransform,
		CacheSizeMax: 1024 * 1024, // 1MB
	})
	db := &Db{diskv: diskv}
	return &ConfigMgr{db: db, users: make(map[string]*UserInfo), dns: make(map[string]*UserInfo)}
}

func NewDiskvStorageBackend(path string) StorageBackend {
	diskv := &DiskvStorageBackend{
		store: diskv.New(diskv.Options{
			BasePath:     path,
			Transform:    transformFunc,
			CacheSizeMax: 0, // no cache
		}),
	}

	return diskv
}

func New(cacheDir string, cacheSize uint64, bucketURL string) *Cache {
	if cacheDir == "" {
		cacheDir, _ = ioutil.TempDir("", "disks3cache")
	}
	dv := diskv.New(diskv.Options{
		BasePath:     cacheDir,
		CacheSizeMax: cacheSize * 1024 * 1024,
	})
	return &Cache{
		disk: diskcache.NewWithDiskv(dv),
		s3:   s3cache.New(bucketURL),
	}
}

func TestKeysNested(t *testing.T) {
	d := diskv.New(diskv.Options{
		BasePath:  "test-data",
		Transform: blockTransform(2),
	})
	defer d.EraseAll()

	for k, v := range keysTestData {
		d.Write(k, []byte(v))
	}

	checkKeys(t, d.Keys(nil), keysTestData)
}

func TestKeysPrefixFlat(t *testing.T) {
	d := diskv.New(diskv.Options{
		BasePath: "test-data",
	})
	defer d.EraseAll()

	for k, v := range keysTestData {
		d.Write(k, []byte(v))
	}

	for _, prefix := range prefixes {
		checkKeys(t, d.KeysPrefix(prefix, nil), filterPrefix(keysTestData, prefix))
	}
}

func (m *ByteStore) Start(config string) error {
	var c Config
	err := json.Unmarshal([]byte(config), &c)
	if err != nil {
		return err
	}

	transformFunc := func(s string) []string {
		return strings.Split(s, SeparationCharacter)
	}
	m.DB = diskv.New(diskv.Options{
		BasePath:     c.RootDir,
		Transform:    transformFunc,
		CacheSizeMax: 1024 * 1024 * c.CacheSize, // in megabytes
	})
	return nil
}

func main() {

	// Set server address
	addr := os.Getenv("ADDRESS")
	if addr == "" {
		log.Fatal("No address provided for the imageproxy")
	}

	// Set cache
	var cache httpcache.Cache
	d := diskv.New(diskv.Options{
		BasePath:     "/tmp/imageproxy",
		CacheSizeMax: 500 * 1024 * 1024,
	})
	cache = diskcache.NewWithDiskv(d)

	// Create proxy
	p := imageproxy.NewProxy(nil, cache)

	// Create whitelist
	if os.Getenv("WHITELIST") != "" {
		p.Whitelist = strings.Split(os.Getenv("WHITELIST"), ",")
	}

	// Create baseurl
	if os.Getenv("BASEURL") != "" {
		var err error
		p.DefaultBaseURL, err = url.Parse(os.Getenv("BASEURL"))
		if err != nil {
			log.Fatalf("error parsing baseURL: %v", err)
		}
	}

	p.ScaleUp = true

	server := &http.Server{
		Addr:    addr,
		Handler: p,
	}

	fmt.Printf("imageproxy listening on " + addr)
	err := server.ListenAndServe()
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func TestKeysFlat(t *testing.T) {
	transform := func(s string) []string {
		if s == "" {
			t.Fatalf(`transform should not be called with ""`)
		}
		return []string{}
	}
	d := diskv.New(diskv.Options{
		BasePath:  "test-data",
		Transform: transform,
	})
	defer d.EraseAll()

	for k, v := range keysTestData {
		d.Write(k, []byte(v))
	}

	checkKeys(t, d.Keys(nil), keysTestData)
}

// setupDb configures the key-value store to which POSTed data will be written,
// rooted in dbDir.
func setupDb(dbDir string) {
	// Based on https://github.com/peterbourgon/diskv/blob/master/examples/content-addressable-store/cas.go#L14
	blockTransform := func(s string) []string {
		sliceSize := len(s) / transformBlockSize
		pathSlice := make([]string, sliceSize)
		for i := 0; i < sliceSize; i++ {
			from, to := i*transformBlockSize, (i*transformBlockSize)+transformBlockSize
			pathSlice[i] = s[from:to]
		}
		return pathSlice
	}

	// Initialize a new diskv store
	db = diskv.New(diskv.Options{
		BasePath: dbDir,
		// Transform:    func(s string) []string { return []string{} },
		Transform:    blockTransform,
		CacheSizeMax: uint64(maxDataSize),
	})
	return
}

func main() {
	d := diskv.New(diskv.Options{
		BasePath:     "my-diskv-data-directory",
		Transform:    func(s string) []string { return []string{} },
		CacheSizeMax: 1024 * 1024, // 1MB
	})

	key := "alpha"
	if err := d.Write(key, []byte{'1', '2', '3'}); err != nil {
		panic(err)
	}

	value, err := d.Read(key)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%v\n", value)

	if err := d.Erase(key); err != nil {
		panic(err)
	}
}