in AdlsDotNetSDK/FileTransfer/FileTransferCommon.cs [304:364]
protected long AddFileToConsumerQueue(string fileFullName, long fileLength, bool isToBeChunked, out long fileSizeToTransfer)
{
string relativePath;
if (SourcePath.Equals(fileFullName)) //This is the case when the input path is a file
{
relativePath = "";
}
else
{
relativePath = fileFullName.Substring(SourcePath.Length);
relativePath = GetDestDirectoryFormatted(relativePath);
}
long numChunks = 0;
string tempChunkFolder = null;
fileSizeToTransfer = 0;
// We will never be here if the log has detected that the file is done
if (isToBeChunked)
{
//If RecordedMetadata.EntryTransferAttemptedLastTime(fileFullName) is true then the file was attempted and incomplete. Because if it is successful then we will not be here
int numChunksAlreadyTransferred = RecordedMetadata.EntryTransferAttemptedLastTime(fileFullName) ? RecordedMetadata.LoadedMetaData[fileFullName].Chunks.Count : -1;
// numChunksAlreadyTransferred is -1 means this file was not attempted before so effectively 0 chunks were done
numChunks = fileLength / ChunkSize + (fileLength % ChunkSize == 0 ? 0 : 1);
tempChunkFolder = RecordedMetadata.EntryTransferAttemptedLastTime(fileFullName) ? RecordedMetadata.LoadedMetaData[fileFullName].SegmentFolder : DestPath + relativePath + Guid.NewGuid() + "Segments";
var bulk = AssignMetaData(fileFullName, tempChunkFolder, DestPath + relativePath, fileLength, numChunks, numChunksAlreadyTransferred);
if (numChunksAlreadyTransferred == numChunks)
{// If all chunks were transferred correctly then add a CopyFileJob only. CopyFileJob will make the necessary checks and determine which state we are in
ConsumerQueue.Add(new CopyFileJob(0, bulk, Client));
}
else
{
// If this file was attempted in last transfer and it is being resumed, at the enumeration time we just add the jobs for the chunks which are not
// reported in the log file. In reality there can be different states 1) Only those reported in resume file are done 2) More than reported are done however concat wasn't started yet
// 3) All chunks are actually done and concat is half done 4) All chunks are done and concat is done. The discrepancy between the resume file and actual events is because the
// log file is written in a separate producer-consumer queue (not serialized) for perf reasons. All these cases checked and are taken care in FileMetaData.
// If we are in state 1 and 2 then the jobs are done as expected. If we are in states beyond 2 then we do the required concat job and chunks
// are reported done without any actual transfer.
for (int i = 0; i < numChunks; i++)
{
if (RecordedMetadata.EntryTransferAttemptedLastTime(fileFullName) && RecordedMetadata.LoadedMetaData[fileFullName].Chunks.Contains(i))
{
continue;
}
ConsumerQueue.Add(new CopyFileJob(i, bulk, Client));
fileSizeToTransfer += i == numChunks - 1 ? fileLength - i * ChunkSize : ChunkSize;
}
}
// Number of chunks to be uploaded for this file will be the total chunks minus the chunks already transferred
numChunks -= numChunksAlreadyTransferred < 0 ? 0 : numChunksAlreadyTransferred;
}
else
{
FileMetaData bulk = AssignMetaData(fileFullName, null, DestPath + relativePath, fileLength, numChunks);
ConsumerQueue.Add(new CopyFileJob(-1, bulk, Client));
fileSizeToTransfer = fileLength;
}
if (FileTransferLog.IsDebugEnabled)
{
FileTransferLog.Debug($"FileTransfer.FileProduced, Name: {fileFullName}, Dest: {tempChunkFolder ?? DestPath + relativePath}, Length: {fileLength}, Chunks: {numChunks}");
}
return numChunks;
}