本文整理汇总了Golang中github.com/aws/aws-sdk-go/service/s3/s3manager.NewDownloader函数的典型用法代码示例。如果您正苦于以下问题:Golang NewDownloader函数的具体用法?Golang NewDownloader怎么用?Golang NewDownloader使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了NewDownloader函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Golang代码示例。
示例1: NewS3
func NewS3(awsId, awsSecret, s3Bucket, s3Region string) Source {
cred := credentials.NewStaticCredentials(awsId, awsSecret, "")
session := session.New(&aws.Config{Region: aws.String(s3Region), Credentials: cred})
s3manager.NewDownloader(session)
return &S3{s3manager.NewDownloader(session), s3Bucket}
}
开发者ID:plimble,项目名称:ivy,代码行数:8,代码来源:s3.go
示例2: downloadFromS3
func downloadFromS3(s3path string, localPath string, scanned func(string, bool)) error {
outputPath := scratchPath(localPath)
sess := session.New(&aws.Config{Region: aws.String("us-west-2")})
downloader := s3manager.NewDownloader(sess, func(d *s3manager.Downloader) {
d.PartSize = 5 * 1024 * 1024 // 5MB per part
})
file, err := os.Create(outputPath)
if err != nil {
return err
}
log.Println("Starting download")
svc := s3.New(sess)
resp, err := svc.HeadObject(&s3.HeadObjectInput{
Bucket: aws.String("smick-media-output"),
Key: aws.String(s3path),
})
if err != nil {
log.Println("Error getting head:", err)
}
log.Println(resp)
downloader.Download(file, &s3.GetObjectInput{
Bucket: aws.String("smick-media-output"),
Key: aws.String(s3path),
}, func(d *s3manager.Downloader) {
log.Println("yeye")
})
log.Println("Downloaded")
return nil
}
开发者ID:omarqazi,项目名称:guide,代码行数:34,代码来源:commands.go
示例3: GetAction
func GetAction(c *cli.Context) {
if len(c.Args()) != 2 {
log.Fatal("get s3path localpath")
}
s3path, err := NewS3Path(c.Args().Get(0))
if err != nil {
log.Fatal(err)
}
localpath := c.Args().Get(1)
if localpath == "" {
log.Fatal("get s3path localpath")
}
log.Printf("s3path Bucket:%v Prefix:%v", s3path.Bucket, s3path.Prefix)
parallel := c.Int("parallel")
manager := s3manager.NewDownloader(nil)
d := NewDownloader(s3path, localpath, parallel, manager)
client := s3.New(nil)
params := &s3.ListObjectsInput{Bucket: &s3path.Bucket, Prefix: &s3path.Prefix}
err = client.ListObjectsPages(params, d.eachPage)
if err != nil {
log.Fatal(err)
}
d.Wait()
}
开发者ID:anarcher,项目名称:s3cli,代码行数:29,代码来源:actions.go
示例4: Get
// Get downloads the specified release from S3 bucket, as described in
// the passed in config, and returns the location of the downloaded
// release and an error, if any
func Get(conf *S3Config) (string, error) {
defaults.DefaultConfig.Region = aws.String(*conf.Region)
tmpDir := os.TempDir()
destFile := filepath.Join(tmpDir, *conf.Revision)
downloadFile, err := os.Create(destFile)
if err != nil {
return "", err
}
filePath := filepath.Join(*conf.RevisionPath, *conf.Revision)
downloader := s3manager.NewDownloader(nil)
_, err = downloader.Download(
downloadFile,
&s3.GetObjectInput{
Bucket: conf.Bucket,
Key: &filePath,
})
if err != nil {
return "", err
}
return destFile, nil
}
开发者ID:ModusCreateOrg,项目名称:peter,代码行数:29,代码来源:s3.go
示例5: getRemoteBackup
// getRemoteBackup is used to pull backups from S3
func getRemoteBackup(r *Restore, conf *config.Config) {
s3Conn := session.New(&aws.Config{Region: aws.String(string(conf.S3Region))})
r.LocalFilePath = fmt.Sprintf("%v/%v", conf.TmpDir, r.RestorePath)
localFileDir := filepath.Dir(r.LocalFilePath)
err := os.MkdirAll(localFileDir, 0755)
if err != nil {
log.Fatalf("[ERR] Unable to create local restore directory!: %v", err)
}
outFile, err := os.Create(r.LocalFilePath)
if err != nil {
log.Fatalf("[ERR] Unable to create local restore temp file!: %v", err)
}
// Create the params to pass into the actual downloader
params := &s3.GetObjectInput{
Bucket: &conf.S3Bucket,
Key: &r.RestorePath,
}
log.Printf("[INFO] Downloading %v%v from S3 in %v", string(conf.S3Bucket), r.RestorePath, string(conf.S3Region))
downloader := s3manager.NewDownloader(s3Conn)
_, err = downloader.Download(outFile, params)
if err != nil {
log.Fatalf("[ERR] Could not download file from S3!: %v", err)
}
outFile.Close()
log.Print("[INFO] Download completed")
}
开发者ID:grubernaut,项目名称:consul-snapshot,代码行数:33,代码来源:restore.go
示例6: GetFile
func (D *Data) GetFile(src string, dst string) error {
log.WithFields(
log.Fields{"src": src,
"dst": dst,
"S3 Data": D,
}).Info("S3: GetFile")
file, err := os.Create(dst)
if err != nil {
log.WithFields(log.Fields{"dst": dst}).Error("S3: Cannot create file")
return err
}
defer file.Close()
s3Param := &s3.GetObjectInput{
Bucket: aws.String(D.BktName),
Key: aws.String(src)}
downloader := s3manager.NewDownloader(D.s3Sess)
n, err := downloader.Download(file, s3Param)
if err != nil {
log.WithFields(
log.Fields{"S3": D, "Key": src, "Error": err}).Error("S3: Cannot download")
os.Remove(dst)
return err
}
log.WithFields(
log.Fields{"S3": D, "Src": src, "Dst": dst, "Bytes recvd": n}).Debug("S3: GetFile")
return nil
}
开发者ID:muthu-r,项目名称:horcrux,代码行数:30,代码来源:s3.go
示例7: DownloadFile
func (client *s3client) DownloadFile(
bucketName string,
remotePath string,
localPath string,
) error {
downloader := s3manager.NewDownloader(client.session)
localFile, err := os.Create(localPath)
if err != nil {
return err
}
defer localFile.Close()
getObject := &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(remotePath),
}
_, err = downloader.Download(localFile, getObject)
if err != nil {
return err
}
return nil
}
开发者ID:pivotal-cf-experimental,项目名称:pivnet-resource,代码行数:25,代码来源:helpers.go
示例8: DownloadFile
func (client *s3client) DownloadFile(bucketName string, remotePath string, versionID string, localPath string) error {
downloader := s3manager.NewDownloader(&s3manager.DownloadOptions{
S3: client.client,
})
localFile, err := os.Create(localPath)
if err != nil {
return err
}
defer localFile.Close()
getObject := &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(remotePath),
}
if versionID != "" {
getObject.VersionId = aws.String(versionID)
}
_, err = downloader.Download(localFile, getObject)
if err != nil {
return err
}
return nil
}
开发者ID:dgodd,项目名称:ssh-resource,代码行数:27,代码来源:s3client.go
示例9: TestDownloadError
func TestDownloadError(t *testing.T) {
s, names, _ := dlLoggingSvc([]byte{1, 2, 3})
opts := &s3manager.DownloadOptions{S3: s, PartSize: 1, Concurrency: 1}
num := 0
s.Handlers.Send.PushBack(func(r *request.Request) {
num++
if num > 1 {
r.HTTPResponse.StatusCode = 400
r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
}
})
d := s3manager.NewDownloader(opts)
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
assert.NotNil(t, err)
assert.Equal(t, int64(1), n)
assert.Equal(t, []string{"GetObject", "GetObject"}, *names)
assert.Equal(t, []byte{1}, w.Bytes())
}
开发者ID:rifflock,项目名称:aws-sdk-go,代码行数:25,代码来源:download_test.go
示例10: validate
func validate(t *testing.T, key string, md5value string) {
mgr := s3manager.NewDownloader(nil)
params := &s3.GetObjectInput{Bucket: bucketName, Key: &key}
w := newDLWriter(1024 * 1024 * 20)
n, err := mgr.Download(w, params)
assert.NoError(t, err)
assert.Equal(t, md5value, fmt.Sprintf("%x", md5.Sum(w.buf[0:n])))
}
开发者ID:jloper3,项目名称:amazon-ecs-cli,代码行数:9,代码来源:integration_test.go
示例11: main
func main() {
jobs := make(chan *s3.Object)
s3Url := os.Args[1]
destDir := os.Args[2]
u, err := url.Parse(s3Url)
if err != nil {
log.Fatal(err)
}
s3Bucket := u.Host
s3Prefix := u.Path[1:]
err = os.MkdirAll(destDir, 0700)
if err != nil {
log.Fatal(err)
}
session := session.New(&aws.Config{Region: aws.String(awsRegion)})
svc := s3.New(session)
downloader := s3manager.NewDownloader(session)
params := &s3.ListObjectsInput{
Bucket: aws.String(s3Bucket),
Prefix: aws.String(s3Prefix),
}
var wg sync.WaitGroup
for w := 1; w <= concurrency; w++ {
wg.Add(1)
go func(w int) {
worker(w, jobs, downloader, s3Bucket, destDir)
defer wg.Done()
}(w)
}
log.Printf("Looking for objects in bucket: %s, prefix: %s", s3Bucket, s3Prefix)
err = svc.ListObjectsPages(params, func(page *s3.ListObjectsOutput, lastPage bool) bool {
for _, object := range page.Contents {
jobs <- object
}
objectsCount += len(page.Contents)
return true
})
close(jobs)
wg.Wait()
if err != nil {
log.Fatal(err)
}
fmt.Printf("Found %d objects to download.\n", objectsCount)
}
开发者ID:johanneswuerbach,项目名称:s3sync,代码行数:56,代码来源:s3sync.go
示例12: NewS3Connection
func NewS3Connection(connection *s3.S3, cacheDirectory string, s3BucketName string,
options *s3manager.DownloadOptions) S3Connection {
conn := S3Connection{
Connection: connection,
BucketName: s3BucketName,
CacheDirectory: cacheDirectory,
}
conn.Downloader = s3manager.NewDownloader(options)
return conn
}
开发者ID:asimihsan,项目名称:arqinator,代码行数:10,代码来源:s3.go
示例13: Download
// Download using to download file from S3 by bucket and key
func Download(session *session.Session, bucket, key, path string) {
file, err := os.Create(path)
logger.Process(err, "Can't create file")
defer file.Close()
client := s3manager.NewDownloader(session)
_, err = client.Download(file, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
logger.Process(err, "Can't retrieve object from S3")
}
开发者ID:artemnikitin,项目名称:s3-tool,代码行数:12,代码来源:download.go
示例14: Download
/*
Download downloads a file from S3.
This is merely a wrapper around the aws-sdk-go downloader. It allows us to
isolate the aws-sdk-go dependencies and unify error handling.
*/
func Download(file *os.File, bucket, key string) error {
downloader := s3manager.NewDownloader(session.New(&aws.Config{Region: aws.String("us-east-1")}))
numBytes, err := downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
return err
}
log.Println("Downloaded", numBytes, "bytes")
return nil
}
开发者ID:venicegeo,项目名称:pzsvc-sdk-go,代码行数:19,代码来源:s3.go
示例15: TestDownloadZero
func TestDownloadZero(t *testing.T) {
s, names, ranges := dlLoggingSvc([]byte{})
opts := &s3manager.DownloadOptions{S3: s}
d := s3manager.NewDownloader(opts)
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
assert.Nil(t, err)
assert.Equal(t, int64(0), n)
assert.Equal(t, []string{"GetObject"}, *names)
assert.Equal(t, []string{"bytes=0-5242879"}, *ranges)
}
开发者ID:rifflock,项目名称:aws-sdk-go,代码行数:16,代码来源:download_test.go
示例16: TestDownloadSetPartSize
func TestDownloadSetPartSize(t *testing.T) {
s, names, ranges := dlLoggingSvc([]byte{1, 2, 3})
opts := &s3manager.DownloadOptions{S3: s, PartSize: 1, Concurrency: 1}
d := s3manager.NewDownloader(opts)
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
assert.Nil(t, err)
assert.Equal(t, int64(3), n)
assert.Equal(t, []string{"GetObject", "GetObject", "GetObject"}, *names)
assert.Equal(t, []string{"bytes=0-0", "bytes=1-1", "bytes=2-2"}, *ranges)
assert.Equal(t, []byte{1, 2, 3}, w.Bytes())
}
开发者ID:rifflock,项目名称:aws-sdk-go,代码行数:17,代码来源:download_test.go
示例17: Fetch
// Fetch fetches target from path specified in opts
func Fetch(opts *Fetcher, target string, showProgress bool) error {
targetPath := filepath.Dir(opts.Destination)
writable, err := targetPathWritable(targetPath)
if !writable || err != nil {
fmt.Printf("Cannot write to target `%s`. Please check that it exists and is writable.\n", targetPath)
return err
}
temp, err := ioutil.TempFile(targetPath, fmt.Sprintf(".%s-", opts.Project))
if err != nil {
return err
}
defer temp.Close()
bar := pb.New64(*targetSize(opts, target)).SetUnits(pb.U_BYTES)
if showProgress {
bar.Start()
}
etag := readMD5Sum(opts.Destination)
writer := &ProgressWriter{temp, bar}
downloader := s3manager.NewDownloader(&s3manager.DownloadOptions{
S3: opts.S3,
})
_, err = downloader.Download(writer, &s3.GetObjectInput{
Bucket: aws.String(opts.Bucket),
Key: opts.Key(target),
IfNoneMatch: aws.String(etag),
})
if err != nil {
os.Remove(temp.Name())
if reqErr, ok := err.(awserr.RequestFailure); ok {
if reqErr.StatusCode() == 304 {
bar.Set64(bar.Total)
bar.FinishPrint("Using local copy.")
return nil
}
return reqErr
}
return err
}
return os.Rename(temp.Name(), opts.Destination)
}
开发者ID:xormplus,项目名称:mhook,代码行数:45,代码来源:mhook.go
示例18: DownloadFile
func (client *s3client) DownloadFile(bucketName string, remotePath string, versionID string, localPath string) error {
headObject := &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(remotePath),
}
if versionID != "" {
headObject.VersionId = aws.String(versionID)
}
object, err := client.client.HeadObject(headObject)
if err != nil {
return err
}
progress := client.newProgressBar(*object.ContentLength)
downloader := s3manager.NewDownloader(client.session)
localFile, err := os.Create(localPath)
if err != nil {
return err
}
defer localFile.Close()
getObject := &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(remotePath),
}
if versionID != "" {
getObject.VersionId = aws.String(versionID)
}
progress.Start()
defer progress.Finish()
_, err = downloader.Download(progressWriterAt{localFile, progress}, getObject)
if err != nil {
return err
}
return nil
}
开发者ID:Pivotal-DataFabric,项目名称:s3-resource,代码行数:44,代码来源:s3client.go
示例19: main
func main() {
flag.Parse()
argsErr := "Missing required arg:"
if *bucket == "" {
log.Fatalln(argsErr, "bucket")
}
if *prefix == "" {
log.Fatalln(argsErr, "prefix")
}
awsSession := session.New(&aws.Config{Region: aws.String(*awsRegion)})
svc := s3.New(awsSession)
loi := &s3.ListObjectsInput{
Bucket: bucket,
Prefix: prefix,
}
listObjectsO, err := svc.ListObjects(loi)
if err != nil {
log.Fatalln("ListObjects failed:", err)
}
if *listObjectsO.IsTruncated {
fmt.Println("ListObjectsOutput is truncated")
}
downloader := s3manager.NewDownloader(awsSession)
fmt.Println("found keys:")
for _, v := range listObjectsO.Contents {
key := *v.Key
fmt.Print("Downloading ", key, " ... ")
file := getFile(key)
_, err := downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(*bucket),
Key: aws.String(key),
})
if err != nil {
log.Fatalln("Failed to download file", err)
}
fmt.Println("done")
file.Close()
}
fmt.Println("DONE")
}
开发者ID:kluyg,项目名称:s3dl,代码行数:42,代码来源:s3dl.go
示例20: GetFileTobeSend
func (s3 *s3driver) GetFileTobeSend(dpconn, dpname, itemlocation, tagdetail string) (filepathname string) {
bucket := getAwsInfoFromDpconn(dpconn)
e := os.MkdirAll(gDpPath+"/"+bucket+"/"+itemlocation, 0777)
if e != nil {
log.Error(e)
return
}
filepathname = gDpPath + "/" + bucket + "/" + itemlocation + "/" + tagdetail
if true == isFileExists(filepathname) {
return
}
//AWS_SECRET_ACCESS_KEY = Env("AWS_SECRET_ACCESS_KEY", false)
//AWS_ACCESS_KEY_ID = Env("AWS_ACCESS_KEY_ID", false)
AWS_REGION = Env("AWS_REGION", false)
file, err := os.Create(filepathname)
if err != nil {
log.Error("Failed to create file", err)
return ""
}
defer file.Close()
downloader := s3manager.NewDownloader(session.New(&aws.Config{Region: aws.String(AWS_REGION)}))
numBytes, err := downloader.Download(file,
&s3aws.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String( /*dpname + "/" + */ itemlocation + "/" + tagdetail),
})
if err != nil {
log.Info("Failed to download file.", err)
os.Remove(filepathname)
return
}
log.Println("Downloaded file", file.Name(), numBytes, "bytes")
return
}
开发者ID:asiainfoLDP,项目名称:datahub,代码行数:42,代码来源:s3driver.go
注:本文中的github.com/aws/aws-sdk-go/service/s3/s3manager.NewDownloader函数示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论