in src/main/java/software/amazon/qldb/tutorial/JournalS3ExportReader.java [68:124]
public static List<JournalBlock> readExport(final DescribeJournalS3ExportResult describeJournalS3ExportResult,
final AmazonS3 amazonS3) {
S3ExportConfiguration exportConfiguration =
describeJournalS3ExportResult.getExportDescription().getS3ExportConfiguration();
ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request()
.withBucketName(exportConfiguration.getBucket())
.withPrefix(exportConfiguration.getPrefix());
ListObjectsV2Result listObjectsV2Result = amazonS3.listObjectsV2(listObjectsRequest);
log.info("Found the following objects for list from s3: ");
listObjectsV2Result.getObjectSummaries()
.forEach(s3ObjectSummary -> log.info(s3ObjectSummary.getKey()));
// Validate initial manifest file was written.
String expectedManifestKey = exportConfiguration.getPrefix() +
describeJournalS3ExportResult.getExportDescription().getExportId() + ".started" + ".manifest";
String initialManifestKey = listObjectsV2Result
.getObjectSummaries()
.stream()
.filter(s3ObjectSummary -> s3ObjectSummary.getKey().equalsIgnoreCase(expectedManifestKey))
.map(S3ObjectSummary::getKey)
.findFirst().orElseThrow(() -> new IllegalStateException("Initial manifest not found."));
log.info("Found the initial manifest with key " + initialManifestKey);
// Find the final manifest file, it should contain the exportId in it.
String completedManifestFileKey = listObjectsV2Result
.getObjectSummaries()
.stream()
.filter(s3ObjectSummary -> s3ObjectSummary.getKey().endsWith("completed.manifest")
&& (s3ObjectSummary
.getKey()
.contains(describeJournalS3ExportResult.getExportDescription().getExportId())))
.map(S3ObjectSummary::getKey)
.findFirst().orElseThrow(() -> new IllegalStateException("Completed manifest not found."));
log.info("Found the completed manifest with key " + completedManifestFileKey);
// Read manifest file to find data file keys.
S3Object completedManifestObject = amazonS3.getObject(exportConfiguration.getBucket(), completedManifestFileKey);
List<String> dataFileKeys = getDataFileKeysFromManifest(completedManifestObject);
log.info("Found the following keys in the manifest files: " + dataFileKeys);
List<JournalBlock> journalBlocks = new ArrayList<>();
for (String key : dataFileKeys) {
log.info("Reading file with S3 key " + key + " from bucket: " + exportConfiguration.getBucket());
S3Object s3Object = amazonS3.getObject(exportConfiguration.getBucket(), key);
List<JournalBlock> blocks = getJournalBlocks(s3Object);
compareKeyWithContentRange(key, blocks.get(0), blocks.get(blocks.size() - 1));
journalBlocks.addAll(blocks);
}
return journalBlocks;
}