def main()

in source/glue-job-scripts/configuration.py [0:0]
60 lines of code
6 McCabe index (conditional complexity)

def main():
    """
    This script creates two CSV files for QuickSight table joining.
    First one contains all information of machines, and the other one contains all configuration of machines.
    """

    # Sets Glue logging
    spark_context = SparkContext()
    glue_context = GlueContext(spark_context)
    logger = glue_context.get_logger()

    # Gets metadata from DynamoDB tables
    logger.info("Gets metadata from DynamoDB tables...")
    metadata = get_medata()
    delimiter = metadata[0]
    location_keys = metadata[1]
    line_keys = metadata[2]

    # Scans all machine information from the UI reference DynamoDB table
    logger.info("Scans all machine information from the UI reference DynamoDB table...")
    ui_reference_scan_items = scan_ui_reference_table()

    # Puts machine information CSV file into the output S3 bucket
    logger.info("Creates machine information CSV file...")
    reference_csv_body = "id,machine_name,location,line"

    split_location_keys = []
    if location_keys:
        split_location_keys = location_keys.split("/")

    split_line_keys = []
    if line_keys:
        split_line_keys = line_keys.split("/")

    for item in ui_reference_scan_items:
        machine_id = item["id"]["S"]
        name = item.get("name", {}).get("S", machine_id)

        split_ids = machine_id.split(delimiter)
        location = build_location_line(split_ids=split_ids, keys=split_location_keys, delimiter=delimiter)
        line = build_location_line(split_ids=split_ids, keys=split_line_keys, delimiter=delimiter)
        reference_csv_body += f"\n'{machine_id}','{name}','{location}','{line}'"

    logger.info("Puts machine information CSV file into the output S3 bucket...")
    put_object_to_s3_bucket(
        body=str.encode(reference_csv_body),
        key=f"{CSV_PREFIX}/{MACHINE_INFORMATION_CSV}"
    )

    # Puts the machine information QuickSight manifest into the output S3 bucket
    logger.info("Puts the machine information QuickSight manifest into the output S3 bucket...")
    machine_information_manifest = get_quicksight_manifest(file_name=MACHINE_INFORMATION_CSV)
    put_object_to_s3_bucket(
        body=str.encode(json.dumps(machine_information_manifest)),
        key=f"{MANIFEST_PREFIX}/{MACHINE_INFORMATION_MANIFEST}"
    )

    # Scans all machine config information from the config DynamoDB table
    logger.info("Scans all machine config information from the config DynamoDB table...")
    config_scan_items = scan_config_table()

    # Puts machine config information CSV file into the output S3 bucket
    logger.info("Creates machine config information CSV file...")
    config_csv_body = "id,status_tag,down_value"
    for item in config_scan_items:
        machine_id = item["id"]["S"]
        status_tag = item.get("machineStatusTagName", {}).get("S", "")
        down_value = item.get("machineStatusDownValue", {}).get("S", "")
        split_down_values = down_value.split(",")

        for val in split_down_values:
            config_csv_body += f"\n'{machine_id}','{status_tag}','{val.strip()}'"

    logger.info("Puts machine config information CSV file into the output S3 bucket...")
    put_object_to_s3_bucket(
        body=str.encode(config_csv_body),
        key=f"{CSV_PREFIX}/{MACHINE_CONFIG_INFORMATION_CSV}"
    )

    # Puts the machine config information QuickSight manifest into the output S3 bucket
    logger.info("Puts the machine config information QuickSight manifest into the output S3 bucket...")
    machine_config_information_manifest = get_quicksight_manifest(file_name=MACHINE_CONFIG_INFORMATION_CSV)
    put_object_to_s3_bucket(
        body=str.encode(json.dumps(machine_config_information_manifest)),
        key=f"{MANIFEST_PREFIX}/{MACHINE_CONFIG_INFORMATION_MANIFEST}"
    )

    logger.info("All done")