Golang s3manager.NewDownloader函数代码示例

原作者: [db:作者] 来自: [db:来源] 收藏邀请

本文整理汇总了Golang中github.com/aws/aws-sdk-go/service/s3/s3manager.NewDownloader函数的典型用法代码示例。如果您正苦于以下问题：Golang NewDownloader函数的具体用法？Golang NewDownloader怎么用？Golang NewDownloader使用的例子？那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。

在下文中一共展示了NewDownloader函数的20个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Golang代码示例。

示例1: NewS3

func NewS3(awsId, awsSecret, s3Bucket, s3Region string) Source {
	cred := credentials.NewStaticCredentials(awsId, awsSecret, "")
	session := session.New(&aws.Config{Region: aws.String(s3Region), Credentials: cred})

	s3manager.NewDownloader(session)

	return &S3{s3manager.NewDownloader(session), s3Bucket}
}

开发者ID:plimble，项目名称:ivy，代码行数:8，代码来源:s3.go

示例2: downloadFromS3

func downloadFromS3(s3path string, localPath string, scanned func(string, bool)) error {
	outputPath := scratchPath(localPath)

	sess := session.New(&aws.Config{Region: aws.String("us-west-2")})
	downloader := s3manager.NewDownloader(sess, func(d *s3manager.Downloader) {
		d.PartSize = 5 * 1024 * 1024 // 5MB per part
	})

	file, err := os.Create(outputPath)
	if err != nil {
		return err
	}

	log.Println("Starting download")

	svc := s3.New(sess)
	resp, err := svc.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String("smick-media-output"),
		Key:    aws.String(s3path),
	})
	if err != nil {
		log.Println("Error getting head:", err)
	}
	log.Println(resp)

	downloader.Download(file, &s3.GetObjectInput{
		Bucket: aws.String("smick-media-output"),
		Key:    aws.String(s3path),
	}, func(d *s3manager.Downloader) {
		log.Println("yeye")
	})
	log.Println("Downloaded")
	return nil
}

开发者ID:omarqazi，项目名称:guide，代码行数:34，代码来源:commands.go

示例3: GetAction

func GetAction(c *cli.Context) {
	if len(c.Args()) != 2 {
		log.Fatal("get s3path localpath")
	}

	s3path, err := NewS3Path(c.Args().Get(0))
	if err != nil {
		log.Fatal(err)
	}

	localpath := c.Args().Get(1)
	if localpath == "" {
		log.Fatal("get s3path localpath")
	}

	log.Printf("s3path Bucket:%v Prefix:%v", s3path.Bucket, s3path.Prefix)
	parallel := c.Int("parallel")

	manager := s3manager.NewDownloader(nil)
	d := NewDownloader(s3path, localpath, parallel, manager)

	client := s3.New(nil)
	params := &s3.ListObjectsInput{Bucket: &s3path.Bucket, Prefix: &s3path.Prefix}
	err = client.ListObjectsPages(params, d.eachPage)
	if err != nil {
		log.Fatal(err)
	}
	d.Wait()
}

开发者ID:anarcher，项目名称:s3cli，代码行数:29，代码来源:actions.go

示例4: Get

// Get downloads the specified release from S3 bucket, as described in
// the passed in config, and returns the location of the downloaded
// release and an error, if any
func Get(conf *S3Config) (string, error) {
	defaults.DefaultConfig.Region = aws.String(*conf.Region)

	tmpDir := os.TempDir()

	destFile := filepath.Join(tmpDir, *conf.Revision)

	downloadFile, err := os.Create(destFile)
	if err != nil {
		return "", err
	}

	filePath := filepath.Join(*conf.RevisionPath, *conf.Revision)
	downloader := s3manager.NewDownloader(nil)
	_, err = downloader.Download(
		downloadFile,
		&s3.GetObjectInput{
			Bucket: conf.Bucket,
			Key:    &filePath,
		})
	if err != nil {
		return "", err
	}

	return destFile, nil
}

开发者ID:ModusCreateOrg，项目名称:peter，代码行数:29，代码来源:s3.go

示例5: getRemoteBackup

// getRemoteBackup is used to pull backups from S3
func getRemoteBackup(r *Restore, conf *config.Config) {
	s3Conn := session.New(&aws.Config{Region: aws.String(string(conf.S3Region))})

	r.LocalFilePath = fmt.Sprintf("%v/%v", conf.TmpDir, r.RestorePath)

	localFileDir := filepath.Dir(r.LocalFilePath)

	err := os.MkdirAll(localFileDir, 0755)
	if err != nil {
		log.Fatalf("[ERR] Unable to create local restore directory!: %v", err)
	}

	outFile, err := os.Create(r.LocalFilePath)
	if err != nil {
		log.Fatalf("[ERR] Unable to create local restore temp file!: %v", err)
	}

	// Create the params to pass into the actual downloader
	params := &s3.GetObjectInput{
		Bucket: &conf.S3Bucket,
		Key:    &r.RestorePath,
	}

	log.Printf("[INFO] Downloading %v%v from S3 in %v", string(conf.S3Bucket), r.RestorePath, string(conf.S3Region))
	downloader := s3manager.NewDownloader(s3Conn)
	_, err = downloader.Download(outFile, params)
	if err != nil {
		log.Fatalf("[ERR] Could not download file from S3!: %v", err)
	}
	outFile.Close()
	log.Print("[INFO] Download completed")
}

开发者ID:grubernaut，项目名称:consul-snapshot，代码行数:33，代码来源:restore.go

示例6: GetFile

func (D *Data) GetFile(src string, dst string) error {
	log.WithFields(
		log.Fields{"src": src,
			"dst":     dst,
			"S3 Data": D,
		}).Info("S3: GetFile")

	file, err := os.Create(dst)
	if err != nil {
		log.WithFields(log.Fields{"dst": dst}).Error("S3: Cannot create file")
		return err
	}
	defer file.Close()

	s3Param := &s3.GetObjectInput{
		Bucket: aws.String(D.BktName),
		Key:    aws.String(src)}
	downloader := s3manager.NewDownloader(D.s3Sess)
	n, err := downloader.Download(file, s3Param)
	if err != nil {
		log.WithFields(
			log.Fields{"S3": D, "Key": src, "Error": err}).Error("S3: Cannot download")
		os.Remove(dst)
		return err
	}

	log.WithFields(
		log.Fields{"S3": D, "Src": src, "Dst": dst, "Bytes recvd": n}).Debug("S3: GetFile")
	return nil
}

开发者ID:muthu-r，项目名称:horcrux，代码行数:30，代码来源:s3.go

示例7: DownloadFile

func (client *s3client) DownloadFile(
	bucketName string,
	remotePath string,
	localPath string,
) error {
	downloader := s3manager.NewDownloader(client.session)

	localFile, err := os.Create(localPath)
	if err != nil {
		return err
	}
	defer localFile.Close()

	getObject := &s3.GetObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(remotePath),
	}

	_, err = downloader.Download(localFile, getObject)
	if err != nil {
		return err
	}

	return nil
}

开发者ID:pivotal-cf-experimental，项目名称:pivnet-resource，代码行数:25，代码来源:helpers.go

示例8: DownloadFile

func (client *s3client) DownloadFile(bucketName string, remotePath string, versionID string, localPath string) error {
	downloader := s3manager.NewDownloader(&s3manager.DownloadOptions{
		S3: client.client,
	})

	localFile, err := os.Create(localPath)
	if err != nil {
		return err
	}
	defer localFile.Close()

	getObject := &s3.GetObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(remotePath),
	}

	if versionID != "" {
		getObject.VersionId = aws.String(versionID)
	}

	_, err = downloader.Download(localFile, getObject)
	if err != nil {
		return err
	}

	return nil
}

开发者ID:dgodd，项目名称:ssh-resource，代码行数:27，代码来源:s3client.go

示例9: TestDownloadError

func TestDownloadError(t *testing.T) {
	s, names, _ := dlLoggingSvc([]byte{1, 2, 3})
	opts := &s3manager.DownloadOptions{S3: s, PartSize: 1, Concurrency: 1}

	num := 0
	s.Handlers.Send.PushBack(func(r *request.Request) {
		num++
		if num > 1 {
			r.HTTPResponse.StatusCode = 400
			r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
		}
	})

	d := s3manager.NewDownloader(opts)
	w := &aws.WriteAtBuffer{}
	n, err := d.Download(w, &s3.GetObjectInput{
		Bucket: aws.String("bucket"),
		Key:    aws.String("key"),
	})

	assert.NotNil(t, err)
	assert.Equal(t, int64(1), n)
	assert.Equal(t, []string{"GetObject", "GetObject"}, *names)
	assert.Equal(t, []byte{1}, w.Bytes())
}

开发者ID:rifflock，项目名称:aws-sdk-go，代码行数:25，代码来源:download_test.go

示例10: validate

func validate(t *testing.T, key string, md5value string) {
	mgr := s3manager.NewDownloader(nil)
	params := &s3.GetObjectInput{Bucket: bucketName, Key: &key}

	w := newDLWriter(1024 * 1024 * 20)
	n, err := mgr.Download(w, params)
	assert.NoError(t, err)
	assert.Equal(t, md5value, fmt.Sprintf("%x", md5.Sum(w.buf[0:n])))
}

开发者ID:jloper3，项目名称:amazon-ecs-cli，代码行数:9，代码来源:integration_test.go

示例11: main

func main() {
	jobs := make(chan *s3.Object)

	s3Url := os.Args[1]
	destDir := os.Args[2]

	u, err := url.Parse(s3Url)
	if err != nil {
		log.Fatal(err)
	}

	s3Bucket := u.Host
	s3Prefix := u.Path[1:]

	err = os.MkdirAll(destDir, 0700)
	if err != nil {
		log.Fatal(err)
	}

	session := session.New(&aws.Config{Region: aws.String(awsRegion)})
	svc := s3.New(session)
	downloader := s3manager.NewDownloader(session)

	params := &s3.ListObjectsInput{
		Bucket: aws.String(s3Bucket),
		Prefix: aws.String(s3Prefix),
	}

	var wg sync.WaitGroup
	for w := 1; w <= concurrency; w++ {
		wg.Add(1)
		go func(w int) {
			worker(w, jobs, downloader, s3Bucket, destDir)
			defer wg.Done()
		}(w)
	}

	log.Printf("Looking for objects in bucket: %s, prefix: %s", s3Bucket, s3Prefix)

	err = svc.ListObjectsPages(params, func(page *s3.ListObjectsOutput, lastPage bool) bool {
		for _, object := range page.Contents {
			jobs <- object
		}
		objectsCount += len(page.Contents)
		return true
	})

	close(jobs)
	wg.Wait()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("Found %d objects to download.\n", objectsCount)
}

开发者ID:johanneswuerbach，项目名称:s3sync，代码行数:56，代码来源:s3sync.go

示例12: NewS3Connection

func NewS3Connection(connection *s3.S3, cacheDirectory string, s3BucketName string,
	options *s3manager.DownloadOptions) S3Connection {
	conn := S3Connection{
		Connection:     connection,
		BucketName:     s3BucketName,
		CacheDirectory: cacheDirectory,
	}
	conn.Downloader = s3manager.NewDownloader(options)
	return conn
}

开发者ID:asimihsan，项目名称:arqinator，代码行数:10，代码来源:s3.go

示例13: Download

// Download using to download file from S3 by bucket and key
func Download(session *session.Session, bucket, key, path string) {
	file, err := os.Create(path)
	logger.Process(err, "Can't create file")
	defer file.Close()
	client := s3manager.NewDownloader(session)
	_, err = client.Download(file, &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	})
	logger.Process(err, "Can't retrieve object from S3")
}

开发者ID:artemnikitin，项目名称:s3-tool，代码行数:12，代码来源:download.go

示例14: Download

/*
Download downloads a file from S3.

This is merely a wrapper around the aws-sdk-go downloader. It allows us to
isolate the aws-sdk-go dependencies and unify error handling.
*/
func Download(file *os.File, bucket, key string) error {
	downloader := s3manager.NewDownloader(session.New(&aws.Config{Region: aws.String("us-east-1")}))
	numBytes, err := downloader.Download(file,
		&s3.GetObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(key),
		})
	if err != nil {
		return err
	}
	log.Println("Downloaded", numBytes, "bytes")
	return nil
}

开发者ID:venicegeo，项目名称:pzsvc-sdk-go，代码行数:19，代码来源:s3.go

示例15: TestDownloadZero

func TestDownloadZero(t *testing.T) {
	s, names, ranges := dlLoggingSvc([]byte{})

	opts := &s3manager.DownloadOptions{S3: s}
	d := s3manager.NewDownloader(opts)
	w := &aws.WriteAtBuffer{}
	n, err := d.Download(w, &s3.GetObjectInput{
		Bucket: aws.String("bucket"),
		Key:    aws.String("key"),
	})

	assert.Nil(t, err)
	assert.Equal(t, int64(0), n)
	assert.Equal(t, []string{"GetObject"}, *names)
	assert.Equal(t, []string{"bytes=0-5242879"}, *ranges)
}

开发者ID:rifflock，项目名称:aws-sdk-go，代码行数:16，代码来源:download_test.go

示例16: TestDownloadSetPartSize

func TestDownloadSetPartSize(t *testing.T) {
	s, names, ranges := dlLoggingSvc([]byte{1, 2, 3})

	opts := &s3manager.DownloadOptions{S3: s, PartSize: 1, Concurrency: 1}
	d := s3manager.NewDownloader(opts)
	w := &aws.WriteAtBuffer{}
	n, err := d.Download(w, &s3.GetObjectInput{
		Bucket: aws.String("bucket"),
		Key:    aws.String("key"),
	})

	assert.Nil(t, err)
	assert.Equal(t, int64(3), n)
	assert.Equal(t, []string{"GetObject", "GetObject", "GetObject"}, *names)
	assert.Equal(t, []string{"bytes=0-0", "bytes=1-1", "bytes=2-2"}, *ranges)
	assert.Equal(t, []byte{1, 2, 3}, w.Bytes())
}

开发者ID:rifflock，项目名称:aws-sdk-go，代码行数:17，代码来源:download_test.go

示例17: Fetch

// Fetch fetches target from path specified in opts
func Fetch(opts *Fetcher, target string, showProgress bool) error {
	targetPath := filepath.Dir(opts.Destination)
	writable, err := targetPathWritable(targetPath)
	if !writable || err != nil {
		fmt.Printf("Cannot write to target `%s`. Please check that it exists and is writable.\n", targetPath)
		return err
	}

	temp, err := ioutil.TempFile(targetPath, fmt.Sprintf(".%s-", opts.Project))
	if err != nil {
		return err
	}
	defer temp.Close()

	bar := pb.New64(*targetSize(opts, target)).SetUnits(pb.U_BYTES)
	if showProgress {
		bar.Start()
	}
	etag := readMD5Sum(opts.Destination)
	writer := &ProgressWriter{temp, bar}

	downloader := s3manager.NewDownloader(&s3manager.DownloadOptions{
		S3: opts.S3,
	})
	_, err = downloader.Download(writer, &s3.GetObjectInput{
		Bucket:      aws.String(opts.Bucket),
		Key:         opts.Key(target),
		IfNoneMatch: aws.String(etag),
	})
	if err != nil {
		os.Remove(temp.Name())
		if reqErr, ok := err.(awserr.RequestFailure); ok {
			if reqErr.StatusCode() == 304 {
				bar.Set64(bar.Total)
				bar.FinishPrint("Using local copy.")
				return nil
			}
			return reqErr
		}
		return err
	}

	return os.Rename(temp.Name(), opts.Destination)
}

开发者ID:xormplus，项目名称:mhook，代码行数:45，代码来源:mhook.go

示例18: DownloadFile

func (client *s3client) DownloadFile(bucketName string, remotePath string, versionID string, localPath string) error {
	headObject := &s3.HeadObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(remotePath),
	}

	if versionID != "" {
		headObject.VersionId = aws.String(versionID)
	}

	object, err := client.client.HeadObject(headObject)
	if err != nil {
		return err
	}

	progress := client.newProgressBar(*object.ContentLength)

	downloader := s3manager.NewDownloader(client.session)

	localFile, err := os.Create(localPath)
	if err != nil {
		return err
	}
	defer localFile.Close()

	getObject := &s3.GetObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(remotePath),
	}

	if versionID != "" {
		getObject.VersionId = aws.String(versionID)
	}

	progress.Start()
	defer progress.Finish()

	_, err = downloader.Download(progressWriterAt{localFile, progress}, getObject)
	if err != nil {
		return err
	}

	return nil
}

开发者ID:Pivotal-DataFabric，项目名称:s3-resource，代码行数:44，代码来源:s3client.go

示例19: main

func main() {
	flag.Parse()
	argsErr := "Missing required arg:"
	if *bucket == "" {
		log.Fatalln(argsErr, "bucket")
	}
	if *prefix == "" {
		log.Fatalln(argsErr, "prefix")
	}

	awsSession := session.New(&aws.Config{Region: aws.String(*awsRegion)})
	svc := s3.New(awsSession)
	loi := &s3.ListObjectsInput{
		Bucket: bucket,
		Prefix: prefix,
	}
	listObjectsO, err := svc.ListObjects(loi)
	if err != nil {
		log.Fatalln("ListObjects failed:", err)
	}
	if *listObjectsO.IsTruncated {
		fmt.Println("ListObjectsOutput is truncated")
	}
	downloader := s3manager.NewDownloader(awsSession)
	fmt.Println("found keys:")
	for _, v := range listObjectsO.Contents {
		key := *v.Key
		fmt.Print("Downloading ", key, " ... ")
		file := getFile(key)
		_, err := downloader.Download(file,
			&s3.GetObjectInput{
				Bucket: aws.String(*bucket),
				Key:    aws.String(key),
			})
		if err != nil {
			log.Fatalln("Failed to download file", err)
		}
		fmt.Println("done")
		file.Close()
	}
	fmt.Println("DONE")
}

开发者ID:kluyg，项目名称:s3dl，代码行数:42，代码来源:s3dl.go

示例20: GetFileTobeSend

func (s3 *s3driver) GetFileTobeSend(dpconn, dpname, itemlocation, tagdetail string) (filepathname string) {

	bucket := getAwsInfoFromDpconn(dpconn)

	e := os.MkdirAll(gDpPath+"/"+bucket+"/"+itemlocation, 0777)
	if e != nil {
		log.Error(e)
		return
	}

	filepathname = gDpPath + "/" + bucket + "/" + itemlocation + "/" + tagdetail

	if true == isFileExists(filepathname) {
		return
	}

	//AWS_SECRET_ACCESS_KEY = Env("AWS_SECRET_ACCESS_KEY", false)
	//AWS_ACCESS_KEY_ID = Env("AWS_ACCESS_KEY_ID", false)
	AWS_REGION = Env("AWS_REGION", false)
	file, err := os.Create(filepathname)
	if err != nil {
		log.Error("Failed to create file", err)
		return ""
	}
	defer file.Close()

	downloader := s3manager.NewDownloader(session.New(&aws.Config{Region: aws.String(AWS_REGION)}))
	numBytes, err := downloader.Download(file,
		&s3aws.GetObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String( /*dpname + "/" + */ itemlocation + "/" + tagdetail),
		})
	if err != nil {
		log.Info("Failed to download file.", err)
		os.Remove(filepathname)
		return
	}

	log.Println("Downloaded file", file.Name(), numBytes, "bytes")

	return
}

开发者ID:asiainfoLDP，项目名称:datahub，代码行数:42，代码来源:s3driver.go

注：本文中的github.com/aws/aws-sdk-go/service/s3/s3manager.NewDownloader函数示例整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。