in cmd/zc_enumerator.go [343:662]
func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context, credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties, includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string, s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel common.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo, stripTopDir bool, trailingDot common.TrailingDotOption, destination *common.Location, excludeContainerNames []string, includeVersionsList bool) (ResourceTraverser, error) {
var output ResourceTraverser
var includeDeleted bool
var includeSnapshot bool
var includeVersion bool
switch permanentDeleteOption {
case common.EPermanentDeleteOption.Snapshots():
includeDeleted = true
includeSnapshot = true
case common.EPermanentDeleteOption.Versions():
includeDeleted = true
includeVersion = true
case common.EPermanentDeleteOption.SnapshotsAndVersions():
includeDeleted = true
includeSnapshot = true
includeVersion = true
}
// print out version id when using azcopy list
if includeVersionsList {
includeVersion = true
}
// Clean up the resource if it's a local path
if location == common.ELocation.Local() {
resource = common.ResourceString{Value: cleanLocalPath(resource.ValueLocal())}
}
// Feed list of files channel into new list traverser
if listOfFilesChannel != nil {
if location.IsLocal() {
// First, ignore all escaped stars. Stars can be valid characters on many platforms (out of the 3 we support though, Windows is the only that cannot support it).
// In the future, should we end up supporting another OS that does not treat * as a valid character, we should turn these checks into a map-check against runtime.GOOS.
tmpResource := common.Iff(runtime.GOOS == "windows", resource.ValueLocal(), strings.ReplaceAll(resource.ValueLocal(), `\*`, ``))
// check for remaining stars. We can't combine list traversers, and wildcarded list traversal occurs below.
if strings.Contains(tmpResource, "*") {
return nil, errors.New("cannot combine local wildcards with include-path or list-of-files")
}
}
output = newListTraverser(resource, location, credential, ctx, recursive, symlinkHandling, getProperties,
listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions, trailingDot, destination)
return output, nil
}
var reauthTok *common.ScopedAuthenticator
if credential != nil {
if at, ok := credential.OAuthTokenInfo.TokenCredential.(common.AuthenticateToken); ok {
// This will cause a reauth with StorageScope, which is fine, that's the original Authenticate call as it stands.
reauthTok = (*common.ScopedAuthenticator)(common.NewScopedCredential(at, common.ECredentialType.OAuthToken()))
}
}
options := createClientOptions(azcopyScanningLogger, nil, reauthTok)
switch location {
case common.ELocation.Local():
_, err := common.OSStat(resource.ValueLocal())
// If wildcard is present and this isn't an existing file/folder, glob and feed the globbed list into a list enum.
if strings.Contains(resource.ValueLocal(), "*") && (stripTopDir || err != nil) {
basePath := getPathBeforeFirstWildcard(resource.ValueLocal())
matches, err := filepath.Glob(resource.ValueLocal())
if err != nil {
return nil, fmt.Errorf("failed to glob: %s", err)
}
globChan := make(chan string)
go func() {
defer close(globChan)
for _, v := range matches {
globChan <- strings.TrimPrefix(v, basePath)
}
}()
baseResource := resource.CloneWithValue(cleanLocalPath(basePath))
output = newListTraverser(baseResource, location, nil, nil, recursive, symlinkHandling, getProperties,
globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions, trailingDot, destination)
} else {
if ctx != nil {
output, _ = newLocalTraverser(*ctx, resource.ValueLocal(), recursive, stripTopDir, symlinkHandling, syncHashType, incrementEnumerationCounter, errorChannel)
} else {
output, _ = newLocalTraverser(context.TODO(), resource.ValueLocal(), recursive, stripTopDir, symlinkHandling, syncHashType, incrementEnumerationCounter, errorChannel)
}
}
case common.ELocation.Benchmark():
ben, err := newBenchmarkTraverser(resource.Value, incrementEnumerationCounter)
if err != nil {
return nil, err
}
output = ben
case common.ELocation.Blob():
// TODO (last service migration) : Remove dependency on URLs.
resourceURL, err := resource.FullURL()
if err != nil {
return nil, err
}
recommendHttpsIfNecessary(*resourceURL)
if ctx == nil {
return nil, errors.New("a valid context must be supplied to create a blob traverser")
}
r := resourceURL.String()
blobURLParts, err := blob.ParseURL(r)
if err != nil {
return nil, err
}
containerName := blobURLParts.ContainerName
// Strip any non-service related things away
blobURLParts.ContainerName = ""
blobURLParts.BlobName = ""
blobURLParts.Snapshot = ""
blobURLParts.VersionID = ""
res, err := SplitResourceString(blobURLParts.String(), common.ELocation.Blob())
if err != nil {
return nil, err
}
c, err := common.GetServiceClientForLocation(common.ELocation.Blob(), res, credential.CredentialType, credential.OAuthTokenInfo.TokenCredential, &options, nil)
if err != nil {
return nil, err
}
bsc, err := c.BlobServiceClient()
if err != nil {
return nil, err
}
if containerName == "" || strings.Contains(containerName, "*") {
if !recursive {
return nil, errors.New(accountTraversalInherentlyRecursiveError)
}
output = newBlobAccountTraverser(bsc, containerName, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions, false, excludeContainerNames)
} else if listOfVersionIds != nil {
output = newBlobVersionsTraverser(r, bsc, *ctx, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions)
} else {
output = newBlobTraverser(r, bsc, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions, false)
}
case common.ELocation.File():
// TODO (last service migration) : Remove dependency on URLs.
resourceURL, err := resource.FullURL()
if err != nil {
return nil, err
}
recommendHttpsIfNecessary(*resourceURL)
if ctx == nil {
return nil, errors.New("a valid context must be supplied to create a file traverser")
}
r := resourceURL.String()
fileURLParts, err := file.ParseURL(r)
if err != nil {
return nil, err
}
shareName := fileURLParts.ShareName
// Strip any non-service related things away
fileURLParts.ShareName = ""
fileURLParts.ShareSnapshot = ""
fileURLParts.DirectoryOrFilePath = ""
fileOptions := &common.FileClientOptions{
AllowTrailingDot: trailingDot.IsEnabled(),
}
res, err := SplitResourceString(fileURLParts.String(), common.ELocation.File())
if err != nil {
return nil, err
}
c, err := common.GetServiceClientForLocation(common.ELocation.File(), res, credential.CredentialType, credential.OAuthTokenInfo.TokenCredential, &options, fileOptions)
if err != nil {
return nil, err
}
fsc, err := c.FileServiceClient()
if err != nil {
return nil, err
}
if shareName == "" || strings.Contains(shareName, "*") {
if !recursive {
return nil, errors.New(accountTraversalInherentlyRecursiveError)
}
output = newFileAccountTraverser(fsc, shareName, *ctx, getProperties, incrementEnumerationCounter, trailingDot, destination)
} else {
output = newFileTraverser(r, fsc, *ctx, recursive, getProperties, incrementEnumerationCounter, trailingDot, destination)
}
case common.ELocation.BlobFS():
resourceURL, err := resource.FullURL()
if err != nil {
return nil, err
}
recommendHttpsIfNecessary(*resourceURL)
if ctx == nil {
return nil, errors.New("a valid context must be supplied to create a blob traverser")
}
r := resourceURL.String()
r = strings.Replace(r, ".dfs", ".blob", 1)
blobURLParts, err := blob.ParseURL(r)
if err != nil {
return nil, err
}
containerName := blobURLParts.ContainerName
// Strip any non-service related things away
blobURLParts.ContainerName = ""
blobURLParts.BlobName = ""
blobURLParts.Snapshot = ""
blobURLParts.VersionID = ""
res, err := SplitResourceString(blobURLParts.String(), common.ELocation.Blob())
if err != nil {
return nil, err
}
c, err := common.GetServiceClientForLocation(common.ELocation.Blob(), res, credential.CredentialType, credential.OAuthTokenInfo.TokenCredential, &options, nil)
if err != nil {
return nil, err
}
bsc, err := c.BlobServiceClient()
if err != nil {
return nil, err
}
includeDirectoryStubs = true // DFS is supposed to feed folders in
if containerName == "" || strings.Contains(containerName, "*") {
if !recursive {
return nil, errors.New(accountTraversalInherentlyRecursiveError)
}
output = newBlobAccountTraverser(bsc, containerName, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions, true, excludeContainerNames)
} else if listOfVersionIds != nil {
output = newBlobVersionsTraverser(r, bsc, *ctx, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions)
} else {
output = newBlobTraverser(r, bsc, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions, true)
}
case common.ELocation.S3():
resourceURL, err := resource.FullURL()
if err != nil {
return nil, err
}
recommendHttpsIfNecessary(*resourceURL)
s3URLParts, err := common.NewS3URLParts(*resourceURL)
if err != nil {
return nil, err
}
if ctx == nil {
return nil, errors.New("a valid context must be supplied to create a S3 traverser")
}
if s3URLParts.BucketName == "" || strings.Contains(s3URLParts.BucketName, "*") {
// TODO convert to path style URL
if !recursive {
return nil, errors.New(accountTraversalInherentlyRecursiveError)
}
output, err = newS3ServiceTraverser(resourceURL, *ctx, getProperties, incrementEnumerationCounter)
if err != nil {
return nil, err
}
} else {
output, err = newS3Traverser(credential.CredentialType, resourceURL, *ctx, recursive, getProperties, incrementEnumerationCounter)
if err != nil {
return nil, err
}
}
case common.ELocation.GCP():
resourceURL, err := resource.FullURL()
if err != nil {
return nil, err
}
recommendHttpsIfNecessary(*resourceURL)
gcpURLParts, err := common.NewGCPURLParts(*resourceURL)
if err != nil {
return nil, err
}
if ctx == nil {
return nil, errors.New("a valid context must be supplied to create a GCP traverser")
}
if gcpURLParts.BucketName == "" || strings.Contains(gcpURLParts.BucketName, "*") {
if !recursive {
return nil, errors.New(accountTraversalInherentlyRecursiveError)
}
output, err = newGCPServiceTraverser(resourceURL, *ctx, getProperties, incrementEnumerationCounter)
if err != nil {
return nil, err
}
} else {
output, err = newGCPTraverser(resourceURL, *ctx, recursive, getProperties, incrementEnumerationCounter)
if err != nil {
return nil, err
}
}
default:
return nil, errors.New("could not choose a traverser from currently available traversers")
}
if output == nil {
panic("sanity check: somehow didn't spawn a traverser")
}
return output, nil
}