in src/Ingestor.cs [1031:1077]
private static IEnumerable<DataSourcesBatch> SplitIntoBatches(IEnumerable<DataSource> objects,
bool localFiles,
long batchSizeLimitInBytes,
int filesPerBatch)
{
bool limitBatchSize = (batchSizeLimitInBytes > 0);
bool limitFilesPerBatch = (filesPerBatch > 0);
int processedFiles = 0;
var ingestionBatches = new List<DataSourcesBatch>();
DataSourcesBatch currentBatch = null;
int runningBatchNumber = 1;
foreach (var f in objects)
{
if (currentBatch == null)
{
currentBatch = new DataSourcesBatch(runningBatchNumber++);
}
else
{
if (!Utilities.EquivalentTimestamps(currentBatch.CreationTimeUtc, f.CreationTimeUtc))
{
ingestionBatches.Add(currentBatch);
currentBatch = new DataSourcesBatch(runningBatchNumber++);
}
}
currentBatch.AddSource(f);
currentBatch.CreationTimeUtc = f.CreationTimeUtc;
processedFiles++;
if ((limitBatchSize && currentBatch.TotalSizeBytes >= batchSizeLimitInBytes) ||
(limitFilesPerBatch && currentBatch.Sources.Count >= filesPerBatch))
{
ingestionBatches.Add(currentBatch);
currentBatch = null;
}
}
if (currentBatch != null)
{
ingestionBatches.Add(currentBatch);
}
return ingestionBatches;
}