in AdlsDotNetSDK/ADLSClient.cs [928:977]
private async Task ConcatenateFilesParallelAsync(string destination, List<string> concatFiles, int recurse, string tempDestination, bool deleteSource = false, CancellationToken cancelToken = default(CancellationToken))
{
if (concatFiles.Count < ConcatenateStreamListThreshold)
{
await ConcatenateFilesAsync(destination, concatFiles, recurse != 0 || deleteSource, cancelToken);
return;
}
int numberTasks = (int)Math.Ceiling((float)concatFiles.Count / ConcatenateStreamListThreshold);
var taskList = new Task[numberTasks];
var destinationList = new List<string>(numberTasks);
// Parallel concat destination files will be as 0,1,2... under tempDetination\GUID-{recurselevel}. And files under tempDestination\Guid will be the input for
// next recursive concat. Adding recurse level tot he temp directory is helpful for debugging purposes
string tempDir = tempDestination + "/" + Guid.NewGuid() + $"-{recurse}";
for (int i = 0; i < numberTasks; i++)
{
destinationList.Add(tempDir + "/" + i);
int start = i * ConcatenateStreamListThreshold;
int count = i < (numberTasks - 1)
? ConcatenateStreamListThreshold
: concatFiles.Count - (numberTasks - 1) * ConcatenateStreamListThreshold;
if (ClientLogger.IsDebugEnabled)
{
ClientLogger.Debug($"Recurse: {recurse}; TaskId: {i}; SourceFiles: {String.Join(",", concatFiles.GetRange(start, count))}; Destination: {destinationList[i]}");
}
// Pass false for recurse!=0 also because softdelete will cleanup the folder anyways
taskList[i] = ConcatenateFilesAsync(destinationList[i], concatFiles.GetRange(start, count),
false, cancelToken);
}
for (int i = 0; i < numberTasks; i++)
{
taskList[i].Wait(cancelToken);
}
// Concatenate is always called with deleteSource as false, because parallel concat jobs cannot delete the source folder
// Now for other recurse levels if you concatenate all the files of a directory softdelete on the SSS will delete the folder since all the files in the tempguid foldler
if (recurse == 0 && deleteSource)
{
string sourcePath = concatFiles[0].Remove(concatFiles[0].LastIndexOf('/'));
if (string.IsNullOrEmpty(sourcePath))
{
throw new ArgumentException("The root directory cant be deleted");
}
await DeleteRecursiveAsync(sourcePath, cancelToken);
}
await ConcatenateFilesParallelAsync(destination, destinationList, recurse + 1, tempDestination, deleteSource, cancelToken);
}