in cmd/zc_traverser_local.go [642:812]
func (t *localTraverser) Traverse(preprocessor objectMorpher, processor objectProcessor, filters []ObjectFilter) (err error) {
singleFileInfo, isSingleFile, err := t.getInfoIfSingleFile()
// it fails here if file does not exist
if err != nil {
azcopyScanningLogger.Log(common.LogError, fmt.Sprintf("Failed to scan path %s: %s", t.fullPath, err.Error()))
return fmt.Errorf("failed to scan path %s due to %w", t.fullPath, err)
}
finalizer, hashingProcessor := t.prepareHashingThreads(preprocessor, processor, filters)
// if the path is a single file, then pass it through the filters and send to processor
if isSingleFile {
if t.incrementEnumerationCounter != nil {
t.incrementEnumerationCounter(common.EEntityType.File())
}
err := processIfPassedFilters(filters,
newStoredObject(
preprocessor,
singleFileInfo.Name(),
"",
common.EEntityType.File(),
singleFileInfo.ModTime(),
singleFileInfo.Size(),
noContentProps, // Local MD5s are computed in the STE, and other props don't apply to local files
noBlobProps,
noMetadata,
"", // Local has no such thing as containers
),
hashingProcessor, // hashingProcessor handles the mutex wrapper
)
_, err = getProcessingError(err)
return finalizer(err)
} else {
if t.recursive {
processFile := func(filePath string, fileInfo os.FileInfo, fileError error) error {
if fileError != nil {
WarnStdoutAndScanningLog(fmt.Sprintf("Accessing %s failed with error: %s", filePath, fileError.Error()))
return nil
}
var entityType common.EntityType
if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
entityType = common.EEntityType.Symlink()
} else if fileInfo.IsDir() {
newFileInfo, err := WrapFolder(filePath, fileInfo)
if err != nil {
WarnStdoutAndScanningLog(fmt.Sprintf("Failed to get last change of target at %s: %s", filePath, err.Error()))
} else {
// fileInfo becomes nil in case we fail to wrap folder.
fileInfo = newFileInfo
}
entityType = common.EEntityType.Folder()
} else {
entityType = common.EEntityType.File()
}
relPath := strings.TrimPrefix(strings.TrimPrefix(cleanLocalPath(filePath), cleanLocalPath(t.fullPath)), common.DeterminePathSeparator(t.fullPath))
if t.symlinkHandling.None() && fileInfo.Mode()&os.ModeSymlink != 0 {
WarnStdoutAndScanningLog(fmt.Sprintf("Skipping over symlink at %s because symlinks are not handled (--follow-symlinks or --preserve-symlinks)", common.GenerateFullPath(t.fullPath, relPath)))
return nil
}
if t.incrementEnumerationCounter != nil {
t.incrementEnumerationCounter(entityType)
}
// This is an exception to the rule. We don't strip the error here, because WalkWithSymlinks catches it.
return processIfPassedFilters(filters,
newStoredObject(
preprocessor,
fileInfo.Name(),
strings.ReplaceAll(relPath, common.DeterminePathSeparator(t.fullPath), common.AZCOPY_PATH_SEPARATOR_STRING), // Consolidate relative paths to the azcopy path separator for sync
entityType,
fileInfo.ModTime(), // get this for both files and folders, since sync needs it for both.
fileInfo.Size(),
noContentProps, // Local MD5s are computed in the STE, and other props don't apply to local files
noBlobProps,
noMetadata,
"", // Local has no such thing as containers
),
hashingProcessor, // hashingProcessor handles the mutex wrapper
)
}
// note: Walk includes root, so no need here to separately create StoredObject for root (as we do for other folder-aware sources)
return finalizer(WalkWithSymlinks(t.appCtx, t.fullPath, processFile, t.symlinkHandling, t.errorChannel))
} else {
// if recursive is off, we only need to scan the files immediately under the fullPath
// We don't transfer any directory properties here, not even the root. (Because the root's
// properties won't be transferred, because the only way to do a non-recursive directory transfer
// is with /* (aka stripTopDir).
entries, err := os.ReadDir(t.fullPath)
if err != nil {
return err
}
entityType := common.EEntityType.File()
// go through the files and return if any of them fail to process
for _, entry := range entries {
// This won't change. It's purely to hand info off to STE about where the symlink lives.
relativePath := entry.Name()
fileInfo, _ := entry.Info()
if fileInfo.Mode()&os.ModeSymlink != 0 {
if t.symlinkHandling.None() {
continue
} else if t.symlinkHandling.Preserve() { // Mark the entity type as a symlink.
entityType = common.EEntityType.Symlink()
} else if t.symlinkHandling.Follow() {
// Because this only goes one layer deep, we can just append the filename to fullPath and resolve with it.
symlinkPath := common.GenerateFullPath(t.fullPath, entry.Name())
// Evaluate the symlink
result, err := UnfurlSymlinks(symlinkPath)
if err != nil {
return err
}
// Resolve the absolute file path of the symlink
result, err = filepath.Abs(result)
if err != nil {
return err
}
// Replace the current FileInfo with
fileInfo, err = common.OSStat(result)
if err != nil {
return err
}
}
}
if entry.IsDir() {
continue
// it doesn't make sense to transfer directory properties when not recurring
}
if t.incrementEnumerationCounter != nil {
t.incrementEnumerationCounter(common.EEntityType.File())
}
err := processIfPassedFilters(filters,
newStoredObject(
preprocessor,
entry.Name(),
strings.ReplaceAll(relativePath, common.DeterminePathSeparator(t.fullPath), common.AZCOPY_PATH_SEPARATOR_STRING), // Consolidate relative paths to the azcopy path separator for sync
entityType, // TODO: add code path for folders
fileInfo.ModTime(),
fileInfo.Size(),
noContentProps, // Local MD5s are computed in the STE, and other props don't apply to local files
noBlobProps,
noMetadata,
"", // Local has no such thing as containers
),
hashingProcessor, // hashingProcessor handles the mutex wrapper
)
_, err = getProcessingError(err)
if err != nil {
return finalizer(err)
}
}
}
}
return finalizer(err)
}