in tst/com/amazon/kinesis/streaming/agent/tailing/AbstractParserTest.java [812:860]
public void testSkipHeadersWithNewFile() throws IOException {
final int numRecords = 100;
final int numHeaders = 2;
flow = spy(flow);
when(flow.getSkipHeaderLines()).thenReturn(numHeaders);
RecordGenerator generator = new RecordGenerator();
P parser = buildParser();
Path testFile1 = testFiles.createTempFile();
generator.setRecordTag("HEADER");
generator.appendRecordsToFile(testFile1, numHeaders);
generator.setRecordTag("DATA");
generator.appendRecordsToFile(testFile1, numRecords);
TrackedFile file1 = new TrackedFile(flow, testFile1);
file1.open(0);
assertTrue(parser.switchParsingToFile(file1));
// Get and validate all records
List<R> records = parseAllRecords(parser, null);
assertEquals(records.size(), numRecords);
assertSkipHeaderLinesRecordsMatchInputFiles(numHeaders, records, testFile1);
for(int i = 0; i < records.size(); ++i) {
assertTrue(ByteBuffers.toString(records.get(i).data(), StandardCharsets.UTF_8).contains("\tDATA\t"));
assertSame(records.get(i).file(), file1);
}
// Continue parsing with a new file and make sure headers are also skipped from second file
int numRecords2 = 200;
Path testFile2 = testFiles.createTempFile();
generator.setRecordTag("HEADER");
generator.appendRecordsToFile(testFile2, numHeaders);
generator.setRecordTag("DATA");
generator.appendRecordsToFile(testFile2, numRecords2);
TrackedFile file2 = new TrackedFile(flow, testFile2);
file2.open(0);
// It shouldn't matter if we continue or switch...
if (TestUtils.decide(0.5))
assertTrue(parser.continueParsingWithFile(file2));
else
assertTrue(parser.switchParsingToFile(file2));
records = parseAllRecords(parser, null);
assertEquals(records.size(), numRecords2);
assertSkipHeaderLinesRecordsMatchInputFiles(numHeaders, records, testFile2);
for(int i = 0; i < records.size(); ++i) {
assertTrue(ByteBuffers.toString(records.get(i).data(), StandardCharsets.UTF_8).contains("\tDATA\t"));
assertSame(records.get(i).file(), file2);
}
}