private static IEnumerable SplitIntoBatches()

in src/Ingestor.cs [1031:1077]


        private static IEnumerable<DataSourcesBatch> SplitIntoBatches(IEnumerable<DataSource> objects,
                                                                      bool localFiles,
                                                                      long batchSizeLimitInBytes,
                                                                      int filesPerBatch)
        {
            bool limitBatchSize = (batchSizeLimitInBytes > 0);
            bool limitFilesPerBatch = (filesPerBatch > 0);

            int processedFiles = 0;
            var ingestionBatches = new List<DataSourcesBatch>();
            DataSourcesBatch currentBatch = null;
            int runningBatchNumber = 1;

            foreach (var f in objects)
            {
                if (currentBatch == null)
                {
                    currentBatch = new DataSourcesBatch(runningBatchNumber++);
                }
                else
                {
                    if (!Utilities.EquivalentTimestamps(currentBatch.CreationTimeUtc, f.CreationTimeUtc))
                    {
                        ingestionBatches.Add(currentBatch);
                        currentBatch = new DataSourcesBatch(runningBatchNumber++);
                    }
                }

                currentBatch.AddSource(f);
                currentBatch.CreationTimeUtc = f.CreationTimeUtc;
                processedFiles++;

                if ((limitBatchSize && currentBatch.TotalSizeBytes >= batchSizeLimitInBytes) ||
                    (limitFilesPerBatch && currentBatch.Sources.Count >= filesPerBatch))
                {
                    ingestionBatches.Add(currentBatch);
                    currentBatch = null;
                }
            }

            if (currentBatch != null)
            {
                ingestionBatches.Add(currentBatch);
            }

            return ingestionBatches;
        }