func (s *S3) init() error {
	if s.Bucket == "" {
		return errors.New("Please Specify an S3 Bucket")
	}
	if s.Region == "" {
		return errors.New("Please Specify an S3 Region")
	}
	if s.Root == "" {
		return errors.New("Please Specify an S3 Root Path")
	}
	if s.BufferDir == "" {
		return errors.New("Please Specify a Buffer Directory to use for Uploads")
	}

	var ok bool
	if s.region, ok = aws.Regions[s.Region]; !ok {
		return errors.New("Invalid Region: " + s.Region)
	}
	err := s.getAuth()
	if err != nil {
		return err
	}
	s.s3 = s3.New(s.auth, s.region)
	s.bucket = s.s3.Bucket(s.Bucket)
	if err := os.MkdirAll(s.BufferDir, 0755); err != nil && !os.IsExist(err) {
		// there was an error and it wasn't that the directory already exists
		return err
	}
	s.bufferDir = &BufferDir{Mutex: sync.Mutex{}, root: s.BufferDir}
	s.root = strings.TrimPrefix(s.Root, "/")
	go s.updateAuthLoop()
	return nil
}

func (app *AppContext) setupS3Logger() (err error) {
	auth, err := aws.GetAuth("", "", "", time.Now())

	if err != nil {
		log.Fatalln("Failed to find AWS credentials in env")
	}

	awsConnection := s3.New(
		auth,
		getAWSRegion(app.config.aws_region),
	)
	bucket := awsConnection.Bucket(app.config.bucket)

	instanceInfo := keygen.BuildInstanceInfo(
		&keygen.EnvInstanceFetcher{},
		serviceName,
		app.config.logging_dir,
	)

	rotateCoordinator := gologging.NewRotateCoordinator(
		app.config.max_log_lines,
		app.config.max_log_age,
	)

	metricsLogger := MetricsLogger{app.metrics}

	app.s3log, err = gologging.StartS3Logger(
		rotateCoordinator,
		instanceInfo,
		&metricsLogger,
		&uploader.S3UploaderBuilder{
			Bucket: bucket,
			KeyNameGenerator: &KeyNameGenerator{
				Info:   instanceInfo,
				Prefix: app.config.key_prefix,
			},
		},
		&metricsLogger,
		app.config.num_workers,
	)
	if err != nil {
		return
	}

	// Make sure logger is flushed when shutdown signal is received
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)
	go func() {
		<-sigc
		log.Println("interrupted, closing logger...")
		app.s3log.Close()
		os.Exit(0)
	}()

	return nil
}

// Deploys a site to S3.
func (s *Site) Deploy(user, pass, url string) error {

	auth := aws.Auth{AccessKey: user, SecretKey: pass}
	b := s3.New(auth, aws.USEast).Bucket(url)

	// walks _site directory and uploads file to S3
	walker := func(fn string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}

		rel, _ := filepath.Rel(s.Dest, fn)
		typ := mime.TypeByExtension(filepath.Ext(rel))
		content, err := ioutil.ReadFile(fn)
		logf(MsgUploadFile, rel)
		if err != nil {
			return err
		}

		// try to upload the file ... sometimes this fails due to amazon
		// issues. If so, we'll re-try
		if err := b.Put(rel, content, typ, s3.PublicRead, s3.Options{}); err != nil {
			time.Sleep(100 * time.Millisecond) // sleep so that we don't immediately retry
			return b.Put(rel, content, typ, s3.PublicRead, s3.Options{})
		}

		// file upload was a success, return nil
		return nil
	}

	return filepath.Walk(s.Dest, walker)
}

// Communicate with all endpoints to see if they are alive.
func (s *ClientTests) TestRegions(c *check.C) {
	errs := make(chan error, len(aws.Regions))
	for _, region := range aws.Regions {
		go func(r aws.Region) {
			s := s3.New(s.s3.Auth, r)
			b := s.Bucket("goamz-" + s.Auth.AccessKey)
			_, err := b.Get("non-existent")
			errs <- err
		}(region)
	}
	for _ = range aws.Regions {
		err := <-errs
		if err != nil {
			s3_err, ok := err.(*s3.Error)
			if ok {
				c.Check(s3_err.Code, check.Matches, "NoSuchBucket")
			} else if _, ok = err.(*net.DNSError); ok {
				// Okay as well.
			} else {
				c.Errorf("Non-S3 error: %s", err)
			}
		} else {
			c.Errorf("Test should have errored but it seems to have succeeded")
		}
	}
}

func s3Setup(bucketName string, path string, opts sequinsOptions) *sequins {
	auth, err := aws.GetAuth(*s3AccessKey, *s3SecretKey, "", time.Time{})
	if err != nil {
		log.Fatal(err)
	}

	regionName := *s3Region
	if regionName == "" {
		regionName = aws.InstanceRegion()
		if regionName == "" {
			log.Fatal("Unspecified --s3-region, and no instance region found.")
		}
	}

	region, exists := aws.Regions[regionName]
	if !exists {
		log.Fatalf("Invalid AWS region: %s", regionName)
	}

	bucket := s3.New(auth, region).Bucket(bucketName)
	backend := backend.NewS3Backend(bucket, path)
	if opts.LocalPath == "" {
		tmpDir, err := ioutil.TempDir("", "sequins-")
		if err != nil {
			log.Fatal(err)
		}

		opts.LocalPath = tmpDir
	}

	return newSequins(backend, opts)
}

func setupS3() *backend.S3Backend {
	fakeS3, _ := s3test.NewServer(&s3test.Config{})

	// cargo-culted from s3test
	fakeRegion := aws.Region{
		Name:                 "faux-region-1",
		S3Endpoint:           fakeS3.URL(),
		S3LocationConstraint: true,
	}

	auth, _ := aws.GetAuth("foo", "bar", "", time.Time{})
	bucket := s3.New(auth, fakeRegion).Bucket("sequinstest")
	bucket.PutBucket("")

	putFile(bucket, "test_data/0/part-00000")
	putFile(bucket, "test_data/0/part-00001")
	putFile(bucket, "test_data/0/_SUCCESS")

	putFile(bucket, "test_data/1/part-00000")
	putFile(bucket, "test_data/1/part-00001")

	bucket.Put("test_data/foo", []byte("nothing"), "", "", s3.Options{})

	return backend.NewS3Backend(bucket, "test_data")
}

func s3Put(config *BackupConfig, local, remote string, public bool) error {
	localf, err := os.Open(local)
	if err != nil {
		return err
	}
	defer localf.Close()
	localfi, err := localf.Stat()
	if err != nil {
		return err
	}

	auth := aws.Auth{AccessKey: config.AwsAccess, SecretKey: config.AwsSecret}
	b := s3.New(auth, aws.USEast).Bucket(config.Bucket)

	acl := s3.Private
	if public {
		acl = s3.PublicRead
	}

	contType := mime.TypeByExtension(path.Ext(local))
	if contType == "" {
		contType = "binary/octet-stream"
	}

	err = b.PutBucket(acl)
	if err != nil {
		return err
	}
	opts := s3.Options{}
	return b.PutReader(remote, localf, localfi.Size(), contType, acl, opts)
}

func SyncFiles(filesToSync []FileToSync) ([]FileProcessed, error) {

	filesProcessedChan := make(chan FileProcessed, len(filesToSync))

	// From IAM or env
	auth, err := GetAWSAuth()
	if err != nil {
		return []FileProcessed{}, err
	}

	region := aws.USWest

	s3_conn := s3.New(auth, region)

	for _, file := range filesToSync {

		go func(file FileToSync) {
			updated := syncFileFromS3(s3_conn, file)
			fileProcessed := FileProcessed{updated, file}
			filesProcessedChan <- fileProcessed
		}(file)

	}

	var filesProcessed []FileProcessed
	for i := 0; i < len(filesToSync); i++ {
		fileProcessed := <-filesProcessedChan
		filesProcessed = append(filesProcessed, fileProcessed)
	}

	return filesProcessed, nil

}

func (s *LocalServerSuite) SetUpSuite(c *gocheck.C) {
	s.srv.SetUp(c)
	s.clientTests.s3 = s3.New(s.srv.auth, s.srv.region)

	// TODO Sadly the fake server ignores auth completely right now. :-(
	s.clientTests.authIsBroken = true
	s.clientTests.Cleanup()
}

func (s *AmazonClientSuite) SetUpSuite(c *check.C) {
	if !testutil.Amazon {
		c.Skip("live tests against AWS disabled (no -amazon)")
	}
	s.srv.SetUp(c)
	s.s3 = s3.New(s.srv.auth, s.Region)
	// In case tests were interrupted in the middle before.
	s.ClientTests.Cleanup()
}

func GetBucket() *s3.Bucket {
	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}

	s := s3.New(auth, aws.USEast)
	return s.Bucket("filmapp-development")
}

func (ss *S3Storage) Connect() error {
	if ss.BucketName == "" {
		return errors.New("S3 bucket name mandatory to establish a connection")
	}

	s3Conn := s3.New(ss.AwsAuth, ss.Region)
	ss.connexion = s3Conn.Bucket(ss.BucketName)

	return nil
}

func main() {
	flag.Parse()
	auth, err := aws.EnvAuth()
	if err != nil {
		log.Fatalln("Failed to recieve auth from env")
	}
	awsConnection := s3.New(
		auth,
		aws.USWest2,
	)
	bucket := awsConnection.Bucket(targetBucket)
	info := gen.BuildInstanceInfo(&localInstanceFetcher{}, "basic_example", ".")
	rotateCoordinator := gologging.NewRotateCoordinator(adjustedMaxLines, time.Hour*1)
	logger, err := gologging.StartS3Logger(
		rotateCoordinator,
		info,
		&stdoutNotifier{},
		&uploader.S3UploaderBuilder{
			Bucket:           bucket,
			KeyNameGenerator: &gen.EdgeKeyNameGenerator{info},
		},
		&stderrNotifier{},
		5,
	)
	if err != nil {
		log.Fatalf("Error building uploader: %s\n ", err)
	}

	i := 0
	now := time.Now()
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)
	go func() {
		<-sigc
		logger.Close()
		fmt.Printf("Produced %f rps\n", float64(i)/(float64(time.Now().Sub(now))/float64(time.Second)))
		os.Exit(0)
	}()
	x := int(time.Second) / *rps
	for ; i < MAX_LINES_PER_LOG*4; i++ {
		// throttle - there is a better solution to this
		defer func() {
			if x := recover(); x != nil { // means we cuagh a signal
				time.Sleep(120 * time.Second)
			}
		}()
		time.Sleep(time.Duration(int(0.8 * float64(x))))
		logger.Log("MOAR! %d", i)
	}
	logger.Close()
}

// create a new s3 client from the url
func newS3Client(config config.Config) (*s3.S3, error) {
	auth, err := aws.GetAuth(config.AWS.AccessKeyID, config.AWS.SecretAccessKey, "", time.Now())
	if err != nil {
		return &s3.S3{}, err
	}

	if config.AWS.Region == "" {
		return nil, errors.New("Region not set for S3 client lib (missing SetS3URL?)")
	}

	return s3.New(auth, aws.Regions[config.AWS.Region]), nil
}

func getS3(u *url.URL) (io.ReadCloser, error) {
	if AWSAuth.AccessKey == "" || AWSRegion.Name == "" {
		return nil, fmt.Errorf("Invalid AWS Auth or Region. Please check env AWS_CONFIG_FILE.")
	}
	client := s3.New(AWSAuth, AWSRegion)
	bucket := client.Bucket(u.Host)
	rc, err := bucket.GetReader(u.Path)
	if err != nil {
		return nil, err
	}
	return rc, nil
}

func GetBucket(bucketPrefix string) (*s3.Bucket, error) {
	auth, err := aws.GetAuth("", "", "", time.Time{})
	if err != nil {
		return nil, err
	}

	s := s3.New(auth, aws.USWest2)
	s.ConnectTimeout = time.Second * 30
	s.ReadTimeout = time.Second * 30

	bucketName := strings.TrimPrefix(bucketPrefix, "s3://") + "-" + environment.GetCloudEnv()
	return s.Bucket(bucketName), nil
}

func (so *s3Options) Execute(args []string) error {
	if err := app.CheckArity(1, 1, args); err != nil {
		return err
	}

	repo := args[0]

	s3 := s3.New(awsAuth, awsRegion)
	buk := s3.Bucket(defBucket)

	data, err := buk.Get("/binary/repos/repositories")

	if err != nil {
		return err
	}

	ts := &env.TagStore{}

	err = json.Unmarshal(data, &ts)

	id, err := ts.Lookup(repo)

	if err != nil {
		return err
	}

	dts, err := env.DefaultTagStore()

	if err != nil {
		return err
	}

	i := &Importer{tags: ts, sysTags: dts}

	if !so.Force {
		if i.alreadyExists(id) {
			return fmt.Errorf("Already have %s, skipping download\n", utils.TruncateID(id))
		}
	}

	fmt.Printf("Downloading %s (%s)\n", repo, utils.TruncateID(id))

	err = i.download(buk, id)

	dts.Flush()

	return err
}

func (d *S3Downstream) Init() error {
	u, err := url.Parse(d.downstreamURI)
	if err != nil || u.Scheme != "s3" || u.User == nil {
		log.Panic("Bad URL scheme ", d.downstreamURI)
	}

	username := u.User.Username()
	password, _ := u.User.Password()

	auth := aws.Auth{AccessKey: username, SecretKey: password}

	log.Println("Init s3 connection using key ", auth.AccessKey, u.Host)
	connection := s3.New(auth, aws.APSoutheast)
	d.bucket = connection.Bucket(u.Host)
	return nil
}

// create a new s3 client from the url
func newS3Client(config config.Config) (*s3.S3, error) {
	auth, err := aws.GetAuth(config.AWS.AccessKeyID, config.AWS.SecretAccessKey, "", time.Now())
	if err != nil {
		return &s3.S3{}, err
	}

	var regionName string

	regQuery := config.AWS.S3URL.Query()["region"]

	if len(regQuery) > 0 && regQuery[0] != "" {
		regionName = regQuery[0]
	} else {
		regionName = S3DefaultRegion
	}

	region := aws.Regions[regionName]

	return s3.New(auth, region), nil
}

func listBackups() {
	bucketName := *config.S3BackupBucket
	dir := sanitizeDirForList(*config.S3BackupDir, bucketDelim)
	auth := aws.Auth{AccessKey: *config.AwsAccess, SecretKey: *config.AwsSecret}
	b := s3.New(auth, aws.USEast).Bucket(bucketName)
	fmt.Printf("Listing files in %s\n", fullUrl(bucketName))
	rsp, err := b.List(dir, bucketDelim, "", 1000)
	if err != nil {
		log.Fatalf("Invalid s3 backup: bucket.List failed %s\n", err)
	}
	//fmt.Printf("rsp: %v\n", rsp)
	if 0 == len(rsp.Contents) {
		fmt.Printf("There are no files in %s\n", fullUrl(*config.S3BackupBucket))
		return
	}
	//fmt.Printf("Backup files in %s:\n", fullUrl(*config.S3BackupBucket))
	for _, key := range rsp.Contents {
		fmt.Printf("  %s %d\n", key.Key, key.Size)
	}
}