def config_runner()

in data_validation/__main__.py [0:0]
55 lines of code
12 McCabe index (conditional complexity)

def config_runner(args):
    """Config Runner is where the decision is made to run validations from one or more files.
    One file can produce multiple validations - for example when more than one set of tables are being validated
    between the source and target. If multiple files are to be run and if the associated configuration files
    are numbered sequentially, say from '0000.yaml' to '0012.yaml' (for 13 validations),
    it is possible to run them concurrently in a Kubernetes / Cloud Run environment.
    If the user wants that, they need to specify a -kc or --kube-completions which tells
    DVT to only run the validation corresponding to the index number provided in the
    JOB_COMPLETION_INDEX (for Kubernetes) or CLOUD_RUN_TASK_INDEX (for Cloud Run) environment
    variable. This environment variable is set by the Kubernetes/Cloud Run container orchestrator.
    The orchestrator spins up containers to complete each validation, one at a time.
    """
    if args.config_dir:
        if args.kube_completions and (
            ("JOB_COMPLETION_INDEX" in os.environ.keys())
            or ("CLOUD_RUN_TASK_INDEX" in os.environ.keys())
        ):
            # Running in Kubernetes in Job completions - only run the yaml file corresponding to index
            job_index = (
                int(os.environ.get("JOB_COMPLETION_INDEX"))
                if "JOB_COMPLETION_INDEX" in os.environ.keys()
                else int(os.environ.get("CLOUD_RUN_TASK_INDEX"))
            )
            config_file_path = (
                f"{args.config_dir}{job_index:04d}.yaml"
                if args.config_dir.endswith("/")
                else f"{args.config_dir}/{job_index:04d}.yaml"
            )
            setattr(args, "config_dir", None)
            setattr(args, "config_file", config_file_path)
            config_managers = build_config_managers_from_yaml(args, config_file_path)
            run_validations(args, config_managers)
        else:
            if args.kube_completions:
                logging.warning(
                    "--kube-completions or -kc specified, however not running in Kubernetes Job completion, check your command line."
                )
            config_file_names = cli_tools.list_validations(config_dir=args.config_dir)
            config_managers = []
            errors = False
            for file in config_file_names:
                config_managers = build_config_managers_from_yaml(args, file)
                try:
                    logging.info(
                        "Currently running the validation for YAML file: %s",
                        file,
                    )
                    run_validations(args, config_managers)
                except Exception as e:
                    errors = True
                    logging.error(
                        "Error '%s' occurred while running config file %s. Skipping it for now.",
                        str(e),
                        file,
                    )
            if errors:
                raise exceptions.ValidationException(
                    "Some of the validations raised an exception"
                )
    else:
        if args.kube_completions:
            logging.warning(
                "--kube-completions or -kc specified, which requires a config directory, however a specific config file is provided."
            )
        config_file_path = _get_arg_config_file(args)
        config_managers = build_config_managers_from_yaml(args, config_file_path)
        run_validations(args, config_managers)