func uploadFileToS3(bucket *s3.Bucket, fpath, s3path string) error {
	// try to get the mime type
	mimetype := ""
	err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
	if err != nil {
		log.Debugf("Magic meme failed for: %v", err)
	} else {
		mimetype, err = magicmime.TypeByFile(fpath)
		if err != nil {
			log.Debugf("Mime type detection for %s failed: %v", fpath, err)
		}
	}

	contents, err := ioutil.ReadFile(fpath)
	if err != nil {
		log.Warnf("Reading %q failed: %v", fpath, err)
	}

	// push the file to s3
	log.Debugf("Pushing %s to s3", s3path)
	if err := bucket.Put(s3path, contents, mimetype, "public-read", s3.Options{CacheControl: "no-cache"}); err != nil {
		return err
	}
	log.Infof("Sucessfully pushed %s to s3", s3path)
	return nil
}

// listFiles lists the files in a specific s3 bucket.
func listFiles(prefix, delimiter, marker string, maxKeys int, b *s3.Bucket) (files []s3.Key, err error) {
	resp, err := b.List(prefix, delimiter, marker, maxKeys)
	if err != nil {
		return nil, err
	}

	// append to files
	for _, fl := range resp.Contents {
		if strings.Contains(fl.Key, "index.html") || strings.Contains(fl.Key, "static") || strings.Contains(fl.Key, "logs") {
			continue
		}

		files = append(files, fl)
	}

	// recursion for the recursion god
	if resp.IsTruncated && resp.NextMarker != "" {
		f, err := listFiles(resp.Prefix, resp.Delimiter, resp.NextMarker, resp.MaxKeys, b)
		if err != nil {
			return nil, err
		}

		// append to files
		files = append(files, f...)
	}

	return files, nil
}

func LastS3KeyWithPrefix(bucket *s3.Bucket, prefix *string) (string, error) {
	narrowedPrefix := *prefix
	keyMarker := ""

	// First, do a few checks for shortcuts for checking backwards: focus in on the 14 days.
	// Otherwise just loop forward until there aren't any more results
	currentDay := time.Now()
	for i := 0; i < S3_REWIND_IN_DAYS_BEFORE_LONG_LOOP; i++ {
		testPrefix := fmt.Sprintf("%s%s", *prefix, S3DatePrefix(&currentDay))
		results, err := bucket.List(narrowedPrefix, "", keyMarker, 0)
		if err != nil && len(results.Contents) > 0 {
			narrowedPrefix = testPrefix
			break
		}
		currentDay = currentDay.Add(-1 * time.Duration(DAY_IN_SECONDS) * time.Second)
	}

	lastKey := ""
	moreResults := true
	for moreResults {
		results, err := bucket.List(narrowedPrefix, "", keyMarker, 0)
		if err != nil {
			return lastKey, err
		}

		if len(results.Contents) == 0 { // empty request, return last found lastKey
			return lastKey, nil
		}

		lastKey = results.Contents[len(results.Contents)-1].Key
		keyMarker = lastKey
		moreResults = results.IsTruncated
	}
	return lastKey, nil
}

func createIndexFile(bucket *s3.Bucket, bucketpath, html string) error {
	p := path.Join(bucketpath, "index.html")
	contents := strings.Replace(index, "{{ . }}", html, -1)

	// push the file to s3
	log.Debugf("Pushing %s to s3", p)
	if err := bucket.Put(p, []byte(contents), "text/html", "public-read", s3.Options{CacheControl: "no-cache"}); err != nil {
		return err
	}
	log.Infof("Sucessfully pushed %s to s3", p)

	return nil
}

func killBucket(b *s3.Bucket) {
	var err error
	for attempt := attempts.Start(); attempt.Next(); {
		err = b.DelBucket()
		if err == nil {
			return
		}
		if _, ok := err.(*net.DNSError); ok {
			return
		}
		e, ok := err.(*s3.Error)
		if ok && e.Code == "NoSuchBucket" {
			return
		}
		if ok && e.Code == "BucketNotEmpty" {
			// Errors are ignored here. Just retry.
			resp, err := b.List("", "", "", 1000)
			if err == nil {
				for _, key := range resp.Contents {
					_ = b.Del(key.Key)
				}
			}
			multis, _, _ := b.ListMulti("", "")
			for _, m := range multis {
				_ = m.Abort()
			}
		}
	}
	message := "cannot delete test bucket"
	if err != nil {
		message += ": " + err.Error()
	}
	panic(message)
}

func CreateManifestInBucket(batch *metadata.LoadBatch, bucket *s3.Bucket) (string, error) {
	manifest, err := makeManifestJson(batch)
	if err != nil {
		return "", err
	}

	url := manifestUrl(bucket.Name, batch.UUID)
	err = bucket.Put(batch.UUID+".json", manifest, "application/json", s3.BucketOwnerRead, s3.Options{})
	if err != nil {
		return "", err
	}

	return url, err
}

func uploadFileToS3(bucket *s3.Bucket, fpath, s3path, contentType string) error {
	contents, err := ioutil.ReadFile(fpath)
	if err != nil {
		return fmt.Errorf("Reading %q failed: %v", fpath, err)
	}

	// push the file to s3
	logrus.Debugf("Pushing %s to s3", s3path)
	if err := bucket.Put(s3path, contents, contentType, "public-read", s3.Options{CacheControl: "no-cache"}); err != nil {
		return err
	}
	logrus.Infof("Sucessfully pushed %s to s3", s3path)
	return nil
}

func (chunkBuffer *ChunkBuffer) StoreToS3AndRelease(s3bucket *s3.Bucket) (bool, error) {
	var s3path string
	var err error

	if debug {
		fmt.Printf("Closing bufferfile: %s\n", chunkBuffer.File.Name())
	}
	chunkBuffer.File.Close()

	contents, err := ioutil.ReadFile(chunkBuffer.File.Name())
	if err != nil {
		return false, err
	}

	if len(contents) <= 0 {
		if debug {
			fmt.Printf("Nothing to store to s3 for bufferfile: %s\n", chunkBuffer.File.Name())
		}
	} else { // Write to s3 in a new filename
		alreadyExists := true
		for alreadyExists {
			writeTime := time.Now()
			s3path = fmt.Sprintf("%s%s%d", S3TopicPartitionPrefix(chunkBuffer.Topic, chunkBuffer.Partition), S3DatePrefix(&writeTime), writeTime.UnixNano())
			alreadyExists, err = s3bucket.Exists(s3path)
			if err != nil {
				panic(err)
				return false, err
			}
		}

		fmt.Printf("S3 Put Object: { Bucket: %s, Key: %s, MimeType:%s }\n", s3bucket.Name, s3path, mime.TypeByExtension(filepath.Ext(chunkBuffer.File.Name())))

		err = s3bucket.Put(s3path, contents, mime.TypeByExtension(filepath.Ext(chunkBuffer.File.Name())), s3.Private, s3.Options{})
		if err != nil {
			panic(err)
		}
	}

	if !keepBufferFiles {
		if debug {
			fmt.Printf("Deleting bufferfile: %s\n", chunkBuffer.File.Name())
		}
		err = os.Remove(chunkBuffer.File.Name())
		if err != nil {
			fmt.Errorf("Error deleting bufferfile %s: %#v", chunkBuffer.File.Name(), err)
		}
	}

	return true, nil
}

func getS3FileHash(bucket *s3.Bucket, filePath string) (string, error) {

	list, err := bucket.List(filePath, "", "", 1)
	if err != nil {
		return "", err
	}

	if len(list.Contents) < 1 {
		return "", ErrFileDoesNotExistS3
	}

	md5sum := strings.Trim(list.Contents[0].ETag, "\"")

	return md5sum, nil

}

func (i *Importer) download(buk *s3.Bucket, id string) error {
	tmpPath := path.Join(env.DIR, "graph", ":artmp:"+id)

	outPath := path.Join(env.DIR, "graph", id)

	os.MkdirAll(tmpPath, 0755)

	os.MkdirAll(path.Join(outPath, "layer"), 0755)

	key := fmt.Sprintf("/binary/repos/%s.layer", id)

	rc, err := buk.GetReader(key)

	if err != nil {
		return err
	}

	cmd := exec.Command("tar", "-f", "-", "-C", tmpPath, "-x")
	cmd.Stdin = rc
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	cmd.Run()

	rc.Close()

	img := i.extract(id, tmpPath)

	if img.Parent != "" {
		if i.alreadyExists(img.Parent) {
			return fmt.Errorf("Parent layer %s already installed, not overwriting\n", img.Parent)
		}

		fmt.Printf("Moving to download parent %s...\n", img.Parent)
		return i.download(buk, img.Parent)
	}

	return nil
}

func putLog(t transfer, bucket *s3.Bucket, dry bool) {
	data, err := ioutil.ReadFile(t.Src)
	if err != nil {
		// Error reading log
		log.Printf("Error reading source file %s:\n", t.Src)
		log.Fatal(err)
	}

	contType := "binary/octet-stream"
	perm := s3.ACL("private")

	if dry {
		log.Printf("Starting sync of %s to bucket path %s...\n", t.Src, t.Dest)
	} else {
		log.Printf("Starting sync of %s to s3://%s/%s...\n", t.Src, bucket.Name, t.Dest)
		err = bucket.Put(t.Dest, data, contType, perm, s3.Options{})
		if err != nil {
			// Error uploading log to s3
			log.Printf("Sync of %s to s3://%s/%s failed:\n", t.Src, bucket.Name, t.Dest)
			log.Fatal(err)
		}
	}
}

func putFile(bucket *s3.Bucket, src string) {
	bytes, _ := ioutil.ReadFile(src)
	bucket.Put(src, bytes, "", "", s3.Options{})
}