func NewS3Session(auth *aws.Auth, region aws.Region) *s3.S3 {
	var s3Session *s3.S3
	cert := x509.Certificate{}
	// go's systemVerify panics with no verify options set
	// TODO: EVG-483
	if runtime.GOOS == "windows" {
		s3Session = s3.New(*auth, region)
		s3Session.ReadTimeout = S3ReadTimeout
		s3Session.WriteTimeout = S3WriteTimeout
		s3Session.ConnectTimeout = S3ConnectTimeout
		return s3Session
	}
	// no verify options so system root ca will be used
	_, err := cert.Verify(x509.VerifyOptions{})
	rootsError := x509.SystemRootsError{}
	if err != nil && err.Error() == rootsError.Error() {
		// create a Transport which includes our TLSConfig with InsecureSkipVerify
		// and client timeouts.
		tlsConfig := tls.Config{InsecureSkipVerify: true}
		tr := http.Transport{
			TLSClientConfig: &tlsConfig}
		// add the Transport to our http client
		client := &http.Client{Transport: &tr}
		s3Session = s3.New(*auth, region, client)
	} else {
		s3Session = s3.New(*auth, region)
	}
	s3Session.ReadTimeout = S3ReadTimeout
	s3Session.WriteTimeout = S3WriteTimeout
	s3Session.ConnectTimeout = S3ConnectTimeout
	return s3Session
}

func NewS3Session(auth *aws.Auth, region aws.Region) *s3.S3 {
	if runtime.GOOS == "darwin" {
		// create a Transport which includes our TLS config
		tlsConfig := tls.Config{InsecureSkipVerify: true}
		tr := http.Transport{TLSClientConfig: &tlsConfig}
		// add the Transport to our http client
		client := &http.Client{Transport: &tr}
		return s3.New(*auth, region, client)
	}
	return s3.New(*auth, region)

}

func uploadImageToS3(path string, fileName string) error {
	fmt.Printf("Filename: %s\n", fileName)

	auth := aws.Auth{
		AccessKey: os.Getenv("ACCESS_KEY"),
		SecretKey: os.Getenv("SECRET_KEY"),
	}

	var region = aws.USEast

	client := s3.New(auth, region)

	data, err := ioutil.ReadFile(fileName)

	if err != nil {
		panic("error reading file! " + fileName)
	}

	bucket := client.Bucket("mesos-hackathon-bucket")
	options := s3.Options{}

	fmt.Printf("Path: %s\n", path)
	err = bucket.Put(path, data, "binary/octet-stream", s3.PublicRead, options)
	if err != nil {
		return err
	}

	return nil
}

func writeFile(f []byte, path string) *model.AppError {

	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
		var auth aws.Auth
		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey

		s := s3.New(auth, awsRegion())
		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)

		ext := filepath.Ext(path)

		var err error
		if model.IsFileExtImage(ext) {
			options := s3.Options{}
			err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options)

		} else {
			options := s3.Options{}
			err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options)
		}

		if err != nil {
			return model.NewAppError("writeFile", "Encountered an error writing to S3", err.Error())
		}
	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
		if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil {
			return err
		}
	} else {
		return model.NewAppError("writeFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
	}

	return nil
}

func TestPutGet(t *testing.T) {
	Reset(t)
	auth, err := aws.EnvAuth()

	if err != nil {
		t.Error(err)
	}

	s := s3.New(auth, localRegion)

	b := s.Bucket("TestBucket")

	err = b.PutBucket("acl")

	if err != nil {
		t.Fatal(err)
	}

	o, err := b.GetBucketContents()

	if err != nil {
		t.Fatal(err)
	}

	if len(*o) != 0 {
		t.Fatalf("Bucket should be empty, but has %d object", len(*o))
	}
}

func moveFile(oldPath, newPath string) *model.AppError {
	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
		fileData := make(chan []byte)
		getFileAndForget(oldPath, fileData)
		fileBytes := <-fileData

		if fileBytes == nil {
			return model.NewLocAppError("moveFile", "api.file.move_file.get_from_s3.app_error", nil, "")
		}

		var auth aws.Auth
		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey

		s := s3.New(auth, awsRegion())
		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)

		if err := bucket.Del(oldPath); err != nil {
			return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
		}

		if err := writeFile(fileBytes, newPath); err != nil {
			return err
		}
	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
		if err := os.Rename(utils.Cfg.FileSettings.Directory+oldPath, utils.Cfg.FileSettings.Directory+newPath); err != nil {
			return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error())
		}
	} else {
		return model.NewLocAppError("moveFile", "api.file.move_file.configured.app_error", nil, "")
	}

	return nil
}

func readFile(path string) ([]byte, *model.AppError) {

	if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
		var auth aws.Auth
		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey

		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)

		// try to get the file from S3 with some basic retry logic
		tries := 0
		for {
			tries++

			f, err := bucket.Get(path)

			if f != nil {
				return f, nil
			} else if tries >= 3 {
				return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error())
			}
			time.Sleep(3000 * time.Millisecond)
		}
	} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
		if f, err := ioutil.ReadFile(utils.Cfg.ServiceSettings.StorageDirectory + path); err != nil {
			return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error())
		} else {
			return f, nil
		}
	} else {
		return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
	}
}

func (rc *Recommender) LoadBackup() (success bool) {
	log.Info("Loading backup from S3:", rc.identifier)
	auth, err := aws.EnvAuth()
	if err != nil {
		log.Error("Problem trying to connect with AWS:", err)
		return false
	}

	s := s3.New(auth, rc.s3Region)
	bucket := s.Bucket(S3BUCKET)

	jsonData, err := bucket.Get(rc.getS3Path())
	if err != nil {
		log.Info("Problem trying to get backup from S3:", err)
		return false
	}

	dataFromJSON := [][]uint64{}
	json.Unmarshal(rc.uncompress(jsonData), &dataFromJSON)

	log.Info("Data loaded from S3:", rc.identifier, "len:", len(dataFromJSON))
	recs := 0
	for _, record := range dataFromJSON {
		scores := make(map[uint64]uint8)
		for i := 1; i < len(record); i += 2 {
			scores[record[i]] = uint8(record[i+1])
		}
		recs += len(scores)
		rc.AddRecord(record[0], scores)
	}

	return true
}

// Communicate with all endpoints to see if they are alive.
func (s *ClientTests) TestRegions(c *gocheck.C) {
	errs := make(chan error, len(aws.Regions))
	for _, region := range aws.Regions {
		go func(r aws.Region) {
			s := s3.New(s.s3.Auth, r)
			b := s.Bucket("goamz-" + s.Auth.AccessKey)
			_, err := b.Get("non-existent")
			errs <- err
		}(region)
	}
	for _ = range aws.Regions {
		err := <-errs
		if err != nil {
			s3_err, ok := err.(*s3.Error)
			if ok {
				c.Check(s3_err.Code, gocheck.Matches, "NoSuchBucket")
			} else if _, ok = err.(*net.DNSError); ok {
				// Okay as well.
			} else {
				c.Errorf("Non-S3 error: %s", err)
			}
		} else {
			c.Errorf("Test should have errored but it seems to have succeeded")
		}
	}
}

func readFile(path string) ([]byte, *model.AppError) {

	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
		var auth aws.Auth
		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey

		s := s3.New(auth, awsRegion())
		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)

		// try to get the file from S3 with some basic retry logic
		tries := 0
		for {
			tries++

			f, err := bucket.Get(path)

			if f != nil {
				return f, nil
			} else if tries >= 3 {
				return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error())
			}
			time.Sleep(3000 * time.Millisecond)
		}
	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
		if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil {
			return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error())
		} else {
			return f, nil
		}
	} else {
		return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
	}
}

func goamzBucket(bucketName, endpoint string) *goamz.Bucket {
	region := aws.Region{
		Name:                 "fake_region",
		S3Endpoint:           s3Server.URL(),
		S3LocationConstraint: true,
	}
	return goamz.New(aws.Auth{}, region).Bucket(bucketName)
}

func (s3p *S3Provider) Version(rawurl, previous string) (string, error) {
	ref := s3p.parse(rawurl)
	resp, err := s3.New(ref.auth, ref.region).Bucket(ref.bucket).Head(ref.path, map[string][]string{})
	if err != nil {
		return "", err
	}
	return resp.Header.Get("x-amz-version-id"), nil
}

func (s *LocalServerSuite) SetUpSuite(c *C) {
	s.srv.SetUp(c)
	s.clientTests.s3 = s3.New(s.srv.auth, s.srv.region)

	// TODO Sadly the fake server ignores auth completely right now. :-(
	s.clientTests.authIsBroken = true
	s.clientTests.Cleanup()
}

func s3Client() *s3.S3 {
	r := aws.Region{
		Name:       "jp-east",
		S3Endpoint: "https://ds.jp-east.idcfcloud.com",
	}
	auth := aws.NewAuth(os.Getenv("IDCF_ACCESS_KEY"), os.Getenv("IDCF_ACCESS_SECRET"), "", time.Now())
	return s3.New(*auth, r)
}

func GetS3File(auth *aws.Auth, s3URL string) (io.ReadCloser, error) {
	urlParsed, err := url.Parse(s3URL)
	if err != nil {
		return nil, err
	}
	session := s3.New(*auth, aws.USEast)
	bucket := session.Bucket(urlParsed.Host)
	return bucket.GetReader(urlParsed.Path)
}

func s3BucketFromConfig(c Config) *s3.Bucket {
	awsAuth := aws.Auth{
		AccessKey: c.AwsClientKey,
		SecretKey: c.AwsSecretKey,
	}
	region := aws.USEast
	connection := s3.New(awsAuth, region)
	return connection.Bucket(c.Bucket)
}

func (s *AmazonClientSuite) SetUpSuite(c *gocheck.C) {
	if !testutil.Amazon {
		c.Skip("live tests against AWS disabled (no -amazon)")
	}
	s.srv.SetUp(c)
	s.s3 = s3.New(s.srv.auth, s.Region)
	// In case tests were interrupted in the middle before.
	s.ClientTests.Cleanup()
}

// NewS3Client returns a new S3 client
func (m *AwsMgr) NewS3Client(accId, region string) (*s3.S3, error) {
	// Get Auth
	auth, err := m.GetAuth(accId)
	if err != nil {
		return nil, err
	}
	s3 := s3.New(*auth, aws.Regions[region])
	return s3, nil
}

func getBucket() (*s3.Bucket, error) {
	auth, err := aws.GetAuth(config.AWS_ACCESS_KEY, config.AWS_SECRET_KEY, "", time.Time{})
	if err != nil {
		return nil, err
	}

	conn := s3.New(auth, aws.Regions["eu-west-1"])
	b := conn.Bucket(config.BUCKET)
	return b, nil
}

func (s3w S3Writer) WriteToS3(p []byte) error {
	auth := aws.Auth{
		AccessKey: s3w.AccessKey,
		SecretKey: s3w.SecretKey,
	}
	path := fmt.Sprintf("%s-%s.tar.gz", s3w.ClusterName, time.Now().UTC().Format(time.RFC3339))

	client := s3.New(auth, aws.Region{S3Endpoint: s3w.Endpoint})
	bucket := client.Bucket(s3w.Bucket)
	return bucket.Put(path, p, "application/x-gzip", s3.Private, s3.Options{})
}