in cmd/copy.go [1433:1655]
func (cca *CookedCopyCmdArgs) processCopyJobPartOrders() (err error) {
ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion)
// Make AUTO default for Azure Files since Azure Files throttles too easily unless user specified concurrency value
if jobsAdmin.JobsAdmin != nil && (cca.FromTo.From() == common.ELocation.File() || cca.FromTo.To() == common.ELocation.File()) && common.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue()) == "" {
jobsAdmin.JobsAdmin.SetConcurrencySettingsToAuto()
}
if err := common.VerifyIsURLResolvable(cca.Source.Value); cca.FromTo.From().IsRemote() && err != nil {
return fmt.Errorf("failed to resolve source: %w", err)
}
if err := common.VerifyIsURLResolvable(cca.Destination.Value); cca.FromTo.To().IsRemote() && err != nil {
return fmt.Errorf("failed to resolve destination: %w", err)
}
// Note: credential info here is only used by remove at the moment.
// TODO: Get the entirety of remove into the new copyEnumeratorInit script so we can remove this
// and stop having two places in copy that we get credential info
// verifies credential type and initializes credential info.
// Note: Currently, only one credential type is necessary for source and destination.
// For upload&download, only one side need credential.
// For S2S copy, as azcopy-v10 use Put*FromUrl, only one credential is needed for destination.
if cca.credentialInfo.CredentialType, err = getCredentialType(ctx, rawFromToInfo{
fromTo: cca.FromTo,
source: cca.Source,
destination: cca.Destination,
}, cca.CpkOptions); err != nil {
return err
}
// For OAuthToken credential, assign OAuthTokenInfo to CopyJobPartOrderRequest properly,
// the info will be transferred to STE.
if cca.credentialInfo.CredentialType.IsAzureOAuth() {
uotm := GetUserOAuthTokenManagerInstance()
// Get token from env var or cache.
if tokenInfo, err := uotm.GetTokenInfo(ctx); err != nil {
return err
} else {
cca.credentialInfo.OAuthTokenInfo = *tokenInfo
}
}
// initialize the fields that are constant across all job part orders,
// and for which we have sufficient info now to set them
jobPartOrder := common.CopyJobPartOrderRequest{
JobID: cca.jobID,
FromTo: cca.FromTo,
ForceWrite: cca.ForceWrite,
ForceIfReadOnly: cca.ForceIfReadOnly,
AutoDecompress: cca.autoDecompress,
Priority: common.EJobPriority.Normal(),
LogLevel: azcopyLogVerbosity,
ExcludeBlobType: cca.excludeBlobType,
SymlinkHandlingType: cca.SymlinkHandling,
BlobAttributes: common.BlobTransferAttributes{
BlobType: cca.blobType,
BlockSizeInBytes: cca.blockSize,
PutBlobSizeInBytes: cca.putBlobSize,
ContentType: cca.contentType,
ContentEncoding: cca.contentEncoding,
ContentLanguage: cca.contentLanguage,
ContentDisposition: cca.contentDisposition,
CacheControl: cca.cacheControl,
BlockBlobTier: cca.blockBlobTier,
PageBlobTier: cca.pageBlobTier,
Metadata: cca.metadata,
NoGuessMimeType: cca.noGuessMimeType,
PreserveLastModifiedTime: cca.preserveLastModifiedTime,
PutMd5: cca.putMd5,
MD5ValidationOption: cca.md5ValidationOption,
DeleteSnapshotsOption: cca.deleteSnapshotsOption,
// Setting tags when tags explicitly provided by the user through blob-tags flag
BlobTagsString: cca.blobTags.ToString(),
DeleteDestinationFileIfNecessary: cca.deleteDestinationFileIfNecessary,
},
CommandString: cca.commandString,
CredentialInfo: cca.credentialInfo,
FileAttributes: common.FileTransferAttributes{
TrailingDot: cca.trailingDot,
},
}
srcCredInfo, err := cca.getSrcCredential(ctx, &jobPartOrder)
if err != nil {
return err
}
var srcReauth *common.ScopedAuthenticator
if at, ok := srcCredInfo.OAuthTokenInfo.TokenCredential.(common.AuthenticateToken); ok {
// This will cause a reauth with StorageScope, which is fine, that's the original Authenticate call as it stands.
srcReauth = (*common.ScopedAuthenticator)(common.NewScopedCredential(at, common.ECredentialType.OAuthToken()))
}
options := createClientOptions(common.AzcopyCurrentJobLogger, nil, srcReauth)
var azureFileSpecificOptions any
if cca.FromTo.From() == common.ELocation.File() {
azureFileSpecificOptions = &common.FileClientOptions{
AllowTrailingDot: cca.trailingDot.IsEnabled(),
}
}
jobPartOrder.SrcServiceClient, err = common.GetServiceClientForLocation(
cca.FromTo.From(),
cca.Source,
srcCredInfo.CredentialType,
srcCredInfo.OAuthTokenInfo.TokenCredential,
&options,
azureFileSpecificOptions,
)
if err != nil {
return err
}
if cca.FromTo.To() == common.ELocation.File() {
azureFileSpecificOptions = &common.FileClientOptions{
AllowTrailingDot: cca.trailingDot.IsEnabled(),
AllowSourceTrailingDot: cca.trailingDot.IsEnabled() && cca.FromTo.From() == common.ELocation.File(),
}
}
var dstReauthTok *common.ScopedAuthenticator
if at, ok := cca.credentialInfo.OAuthTokenInfo.TokenCredential.(common.AuthenticateToken); ok {
// This will cause a reauth with StorageScope, which is fine, that's the original Authenticate call as it stands.
dstReauthTok = (*common.ScopedAuthenticator)(common.NewScopedCredential(at, common.ECredentialType.OAuthToken()))
}
var srcCred *common.ScopedToken
if cca.FromTo.IsS2S() && srcCredInfo.CredentialType.IsAzureOAuth() {
srcCred = common.NewScopedCredential(srcCredInfo.OAuthTokenInfo.TokenCredential, srcCredInfo.CredentialType)
}
options = createClientOptions(common.AzcopyCurrentJobLogger, srcCred, dstReauthTok)
jobPartOrder.DstServiceClient, err = common.GetServiceClientForLocation(
cca.FromTo.To(),
cca.Destination,
cca.credentialInfo.CredentialType,
cca.credentialInfo.OAuthTokenInfo.TokenCredential,
&options,
azureFileSpecificOptions,
)
if err != nil {
return err
}
jobPartOrder.DestinationRoot = cca.Destination
jobPartOrder.SourceRoot = cca.Source
jobPartOrder.SourceRoot.Value, err = GetResourceRoot(cca.Source.Value, cca.FromTo.From())
if err != nil {
return err
}
// Stripping the trailing /* for local occurs much later than stripping the trailing /* for remote resources.
// TODO: Move these into the same place for maintainability.
if diff := strings.TrimPrefix(cca.Source.Value, jobPartOrder.SourceRoot.Value); cca.FromTo.From().IsLocal() &&
diff == "*" || diff == common.OS_PATH_SEPARATOR+"*" || diff == common.AZCOPY_PATH_SEPARATOR_STRING+"*" {
// trim the /*
cca.Source.Value = jobPartOrder.SourceRoot.Value
// set stripTopDir to true so that --list-of-files/--include-path play nice
cca.StripTopDir = true
}
// TODO: Remove this check when FileBlob w/ File OAuth works.
if cca.FromTo.IsS2S() && cca.FromTo.From() == common.ELocation.File() && srcCredInfo.CredentialType.IsAzureOAuth() && cca.FromTo.To() != common.ELocation.File() {
return fmt.Errorf("S2S copy from Azure File authenticated with Azure AD to Blob/BlobFS is not supported")
}
// Check if destination is system container
if cca.FromTo.IsS2S() || cca.FromTo.IsUpload() {
dstContainerName, err := GetContainerName(cca.Destination.Value, cca.FromTo.To())
if err != nil {
return fmt.Errorf("failed to get container name from destination (is it formatted correctly?): %w", err)
}
if common.IsSystemContainer(dstContainerName) {
return fmt.Errorf("cannot copy to system container '%s'", dstContainerName)
}
}
switch {
case cca.FromTo.IsUpload(), cca.FromTo.IsDownload(), cca.FromTo.IsS2S():
// Execute a standard copy command
var e *CopyEnumerator
e, err = cca.initEnumerator(jobPartOrder, srcCredInfo, ctx)
if err != nil {
return fmt.Errorf("failed to initialize enumerator: %w", err)
}
err = e.enumerate()
case cca.FromTo.IsDelete():
// Delete gets ran through copy, so handle delete
if cca.FromTo.From() == common.ELocation.BlobFS() {
// TODO merge with BlobTrash case
// Currently, Blob Delete in STE does not appropriately handle folders. In addition, dfs delete is free-ish.
err = removeBfsResources(cca)
} else {
e, createErr := newRemoveEnumerator(cca)
if createErr != nil {
return fmt.Errorf("failed to initialize enumerator: %w", createErr)
}
err = e.enumerate()
}
case cca.FromTo.IsSetProperties():
// Set properties as well
e, createErr := setPropertiesEnumerator(cca)
if createErr != nil {
return fmt.Errorf("failed to initialize enumerator: %w", createErr)
}
err = e.enumerate()
default:
return fmt.Errorf("copy direction %v is not supported\n", cca.FromTo)
}
if err != nil {
if err == NothingToRemoveError || err == NothingScheduledError {
return err // don't wrap it with anything that uses the word "error"
} else {
return fmt.Errorf("cannot start job due to error: %s.\n", err)
}
}
return nil
}