in dth/job.go [183:213]
func (f *Finder) getTargetObjects(ctx context.Context, prefix *string) (objects map[string]*int64) {
destPrefix := appendPrefix(prefix, &f.cfg.DestPrefix)
log.Printf("Scanning in destination prefix /%s\n", *destPrefix)
token := ""
objects = make(map[string]*int64, 1<<17)
i := 0
batch := 10
for token != "End" {
tar, err := f.desClient.ListObjects(ctx, &token, destPrefix, f.cfg.MaxKeys)
if err != nil {
log.Fatalf("Error listing objects in destination bucket - %s\n", err.Error())
}
// fmt.Printf("Size is %d\n", len(jobs))
// fmt.Printf("Token is %s\n", token)
for _, obj := range tar {
// fmt.Printf("key is %s, size is %d\n", job.Key, job.Size)
srcKey := removePrefix(&obj.Key, &f.cfg.DestPrefix)
objects[*srcKey] = &obj.Size
}
i++
if (i % batch) == 0 {
log.Printf("Scanned %d objects...", i*1000)
}
}
log.Printf("Totally %d objects in destination prefix /%s\n", len(objects), *destPrefix)
return
}