in index.js [400:441]
function getBatchRanges(records) {
var batches = [];
var currentLowOffset = 0;
var batchCurrentBytes = 0;
var batchCurrentCount = 0;
var recordSize;
var nextRecordSize;
for (var i = 0; i < records.length; i++) {
// need to calculate the total record size for the call to Firehose on
// the basis of of non-base64 encoded values
recordSize = Buffer.byteLength(records[i].toString(targetEncoding), targetEncoding);
// batch always has 1 entry, so add it first
batchCurrentBytes += recordSize;
batchCurrentCount += 1;
// To get next record size inorder to calculate the FIREHOSE_MAX_BATCH_BYTES
if(i === records.length - 1) {
nextRecordSize = 0;
} else {
nextRecordSize = Buffer.byteLength(records[i+1].toString(targetEncoding), targetEncoding);
}
// generate a new batch marker every 4MB or 500 records, whichever comes
// first
if (batchCurrentCount === FIREHOSE_MAX_BATCH_COUNT || batchCurrentBytes + nextRecordSize > FIREHOSE_MAX_BATCH_BYTES || i === records.length - 1) {
batches.push({
lowOffset: currentLowOffset,
// annoying special case handling for record sets of size 1
highOffset: i + 1,
sizeBytes: batchCurrentBytes
});
// reset accumulators
currentLowOffset = i + 1;
batchCurrentBytes = 0;
batchCurrentCount = 0;
}
}
return batches;
}