in dth/client.go [319:356]
func (c *S3Client) ListSelectedPrefixes(ctx context.Context, key *string) (prefixes []*string) {
downloader := manager.NewDownloader(c.client)
getBuf := manager.NewWriteAtBuffer([]byte{})
input := &s3.GetObjectInput{
Bucket: &c.bucket,
Key: key,
}
dounload_start := time.Now()
log.Printf("Start downloading the Prefix List File.")
_, err := downloader.Download(ctx, getBuf, input)
download_end := time.Since(dounload_start)
if err != nil {
fmt.Print(err)
} else {
log.Printf("Download the Prefix List File Completed in %v\n", download_end)
}
start := time.Now()
prefixes_value := make([]string, 0, 100000000)
for i, m := range strings.Split(string(getBuf.Bytes()), "\n") {
if i > 100000000 {
log.Printf("The number of prefixes in the list file is larger than 100,000,000, please seperate the file.")
return
}
if len(m) > 0 {
prefixes_value = append(prefixes_value, m)
prefixes = append(prefixes, &prefixes_value[i])
}
}
log.Printf("Got %d prefixes from the customized list file.", len(prefixes))
end := time.Since(start)
log.Printf("Getting Prefixes List Job Completed in %v\n", end)
return
}