def read_export()

in pyqldbsamples/journal_s3_export_reader.py [0:0]

• 25 lines of code
• 3 McCabe index (conditional complexity)

def read_export(describe_journal_export_result, s3_client):
    """
    Read the S3 export within a journal block.

    :type describe_journal_export_result: dict
    :param describe_journal_export_result: The result from QLDB describing a journal export.

    :type s3_client: :py:class:`botocore.client.BaseClient`
    :param s3_client: The low-level S3 client.

    :rtype: list
    :return: List of journal blocks.
    """
    export_configuration = describe_journal_export_result.get('S3ExportConfiguration')
    prefix = export_configuration.get('Prefix')
    bucket_name = export_configuration.get('Bucket')
    response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=prefix)
    objects = response['Contents']
    logger.info('Found the following objects for list from S3:')
    for obj in objects:
        logger.info(obj['Key'])

    # Validate initial manifest file was written
    expected_manifest_key = "{}{}.started.manifest".format(prefix, describe_journal_export_result.get('ExportId'))
    initial_manifest = filter_for_initial_manifest(objects, expected_manifest_key)
    logger.info('Found the initial manifest with key: {}.'.format(initial_manifest))

    # Find the final manifest file, it should contain the exportId in it.
    completed_manifest_file_key = filter_for_completed_manifest(objects)
    completed_manifest_object = s3_client.get_object(Bucket=bucket_name, Key=completed_manifest_file_key)['Body']\
        .read().decode('utf-8')

    data_file_keys = get_data_file_keys_from_manifest(completed_manifest_object)

    logger.info('Found the following keys in the manifest files: {}.'.format(json_dumps(data_file_keys, indent=4)))
    journal_blocks = []
    for key in data_file_keys:
        logger.info('Reading file with S3 key {} from bucket: {}.'.format(key, bucket_name))
        s3_object = s3_client.get_object(Bucket=bucket_name, Key=key)['Body'].read().decode('utf-8')
        blocks = get_journal_blocks(s3_object)
        compare_key_with_content_range(key, blocks[0], blocks[len(blocks) - 1])
        journal_blocks.append(blocks)
    return journal_blocks