public static List readExport()

in src/main/java/software/amazon/qldb/tutorial/JournalS3ExportReader.java [68:124]


    public static List<JournalBlock> readExport(final DescribeJournalS3ExportResult describeJournalS3ExportResult,
        final AmazonS3 amazonS3) {

        S3ExportConfiguration exportConfiguration =
            describeJournalS3ExportResult.getExportDescription().getS3ExportConfiguration();

        ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request()
            .withBucketName(exportConfiguration.getBucket())
            .withPrefix(exportConfiguration.getPrefix());

        ListObjectsV2Result listObjectsV2Result = amazonS3.listObjectsV2(listObjectsRequest);
        log.info("Found the following objects for list from s3: ");
        listObjectsV2Result.getObjectSummaries()
            .forEach(s3ObjectSummary -> log.info(s3ObjectSummary.getKey()));

        // Validate initial manifest file was written.
        String expectedManifestKey = exportConfiguration.getPrefix() +
                describeJournalS3ExportResult.getExportDescription().getExportId() + ".started" + ".manifest";
        String initialManifestKey = listObjectsV2Result
            .getObjectSummaries()
            .stream()
            .filter(s3ObjectSummary -> s3ObjectSummary.getKey().equalsIgnoreCase(expectedManifestKey))
            .map(S3ObjectSummary::getKey)
            .findFirst().orElseThrow(() -> new IllegalStateException("Initial manifest not found."));

        log.info("Found the initial manifest with key " + initialManifestKey);

        // Find the final manifest file, it should contain the exportId in it.
        String completedManifestFileKey = listObjectsV2Result
            .getObjectSummaries()
            .stream()
            .filter(s3ObjectSummary -> s3ObjectSummary.getKey().endsWith("completed.manifest")
                    && (s3ObjectSummary
                    .getKey()
                    .contains(describeJournalS3ExportResult.getExportDescription().getExportId())))
            .map(S3ObjectSummary::getKey)
            .findFirst().orElseThrow(() -> new IllegalStateException("Completed manifest not found."));

        log.info("Found the completed manifest with key " + completedManifestFileKey);

        // Read manifest file to find data file keys.
        S3Object completedManifestObject = amazonS3.getObject(exportConfiguration.getBucket(), completedManifestFileKey);

        List<String> dataFileKeys = getDataFileKeysFromManifest(completedManifestObject);

        log.info("Found the following keys in the manifest files: " + dataFileKeys);

        List<JournalBlock> journalBlocks = new ArrayList<>();
        for (String key : dataFileKeys) {
            log.info("Reading file with S3 key " + key + " from bucket: " + exportConfiguration.getBucket());
            S3Object s3Object = amazonS3.getObject(exportConfiguration.getBucket(), key);
            List<JournalBlock> blocks = getJournalBlocks(s3Object);
            compareKeyWithContentRange(key, blocks.get(0), blocks.get(blocks.size() - 1));
            journalBlocks.addAll(blocks);
        }
        return journalBlocks;
    }