def s3_sync_results()

in service/app/engine.py [0:0]

• 21 lines of code
• 8 McCabe index (conditional complexity)

def s3_sync_results(bucket, prefix, local_dir):

    logging.warning(f"Sending data from {local_dir} to {bucket}/{prefix}")
    files = absolute_file_paths(local_dir)
    count = 0
    for local_path in files:
        logging.warning(f"file found: {local_path}")
        if (
            local_path.endswith(".parquet")
            or local_path.endswith(".png")
            or local_path.endswith(".mp4")
            or local_path.endswith(".info")
            or local_path.endswith(".log")
        ):

            f = local_path.split(local_dir)[-1]
            s3_path = os.path.join(prefix, f)
            logging.warning("Uploading " + local_path + " to " + s3_path)
            success = upload_file(local_path, bucket, object_name=s3_path)
            if not success:
                raise ClientError("Failed to upload to s3")
            count = count + 1
    print(f"Uploaded {count} files to S3")