private async Task ConcatenateFilesParallelAsync()

in AdlsDotNetSDK/ADLSClient.cs [928:977]


        private async Task ConcatenateFilesParallelAsync(string destination, List<string> concatFiles, int recurse, string tempDestination, bool deleteSource = false, CancellationToken cancelToken = default(CancellationToken))
        {
            if (concatFiles.Count < ConcatenateStreamListThreshold)
            {
                await ConcatenateFilesAsync(destination, concatFiles, recurse != 0 || deleteSource, cancelToken);
                return;
            }

            int numberTasks = (int)Math.Ceiling((float)concatFiles.Count / ConcatenateStreamListThreshold);
            var taskList = new Task[numberTasks];
            var destinationList = new List<string>(numberTasks);
            // Parallel concat destination files will be as 0,1,2... under tempDetination\GUID-{recurselevel}. And files under tempDestination\Guid will be the input for 
            // next recursive concat. Adding recurse level tot he temp directory is helpful for debugging purposes
            string tempDir = tempDestination + "/" + Guid.NewGuid() + $"-{recurse}";
            for (int i = 0; i < numberTasks; i++)
            {
                destinationList.Add(tempDir + "/" + i);
                int start = i * ConcatenateStreamListThreshold;
                int count = i < (numberTasks - 1)
                    ? ConcatenateStreamListThreshold
                    : concatFiles.Count - (numberTasks - 1) * ConcatenateStreamListThreshold;
                if (ClientLogger.IsDebugEnabled)
                {
                    ClientLogger.Debug($"Recurse: {recurse}; TaskId: {i}; SourceFiles: {String.Join(",", concatFiles.GetRange(start, count))}; Destination: {destinationList[i]}");
                }
                // Pass false for recurse!=0 also because softdelete will cleanup the folder anyways
                taskList[i] = ConcatenateFilesAsync(destinationList[i], concatFiles.GetRange(start, count),
                    false, cancelToken);
            }

            for (int i = 0; i < numberTasks; i++)
            {
                taskList[i].Wait(cancelToken);
            }

            // Concatenate is always called with deleteSource as false, because parallel concat jobs cannot delete the source folder
            // Now for other recurse levels if you concatenate all the files of a directory softdelete on the SSS will delete the folder since all the files in the tempguid foldler
            if (recurse == 0 && deleteSource)
            {
                string sourcePath = concatFiles[0].Remove(concatFiles[0].LastIndexOf('/'));
                if (string.IsNullOrEmpty(sourcePath))
                {
                    throw new ArgumentException("The root directory cant be deleted");
                }

                await DeleteRecursiveAsync(sourcePath, cancelToken);
            }

            await ConcatenateFilesParallelAsync(destination, destinationList, recurse + 1, tempDestination, deleteSource, cancelToken);
        }