in plugin/source/s3/s3.go [67:128]
func (s *Source) DownloadToPath(dlPath string) error {
if err := os.MkdirAll(dlPath, 0755); err != nil {
return err
}
s.logger.Debugf(0, "copy directory %s is ready", dlPath)
/*
AWS download:
- build credentials and init session
- create temporary file in dlPath
- download s3 data to temp file
- rename temporary file to output file
- if archive: decompress to dlPath
*/
cfg := aws.NewConfig().WithRegion(s.Region)
if s.Credentials.AccessKeyID != "" && s.Credentials.SecretAccessKey != "" {
cfg = cfg.WithCredentials(
credentials.NewStaticCredentials(s.Credentials.AccessKeyID, s.Credentials.SecretAccessKey, ""),
)
}
sess, err := session.NewSession(cfg)
if err != nil {
s.logger.Debugf(0, "failed to create AWS session: %s", err)
return err
}
dl := s3manager.NewDownloader(sess)
outfn := filepath.Join(dlPath, filepath.Base(s.Key))
// create tmpfile in dlPath to store S3 contents before Renaming to final location
tmpfh, err := ioutil.TempFile(dlPath, "")
if err != nil {
s.logger.Debugf(0, "failed to create output file for S3 download: %s", err)
return err
}
defer tmpfh.Close()
n, err := dl.Download(tmpfh, &s3.GetObjectInput{
Bucket: &s.Bucket,
Key: &s.Key,
})
if err != nil {
s.logger.Debugf(0, "failed to download data from S3: %s", err)
return err
}
s.logger.Debugf(0, "downloaded %d bytes for %s:%s from S3", n, s.Bucket, s.Key)
tmpfh.Close()
if err := os.Rename(tmpfh.Name(), outfn); err != nil {
s.logger.Errorf("failed to relocate", outfn, dlPath)
return err
}
s.logger.Debugf(0, "relocated downloaded file from %s to %s", tmpfh.Name(), outfn)
if s.Archive {
if err := archiver.Unarchive(outfn, dlPath); err != nil {
s.logger.Errorf("failed to unarchive %s to dir %s", outfn, dlPath)
return err
}
}
return nil
}