def __init__()

in lmms_eval/api/task.py [0:0]

• 70 lines of code
• 27 McCabe index (conditional complexity)

    def __init__(self, model_name) -> None:  # TODO no super() call here
        # Get pre-configured attributes
        self._config = self.CONFIG
        # different model requires different prompt, we have to take those into account.

        self.model_name = model_name
        self._prepare_model_specific_config()

        assert self.config.output_type in ALL_OUTPUT_TYPES
        self.OUTPUT_TYPE = self.config.output_type

        self.DATASET_PATH = self.config.dataset_path

        if self.config.dataset_name is not None:
            self.DATASET_NAME = self.config.dataset_name

        self._prepare_metric_and_aggregation()

        self.download(self.config.dataset_kwargs)
        self._training_docs = None
        self._fewshot_docs = None

        if self.config.filter_list is not None:
            self._filters = []
            for filter_config in self.config.filter_list:
                for filter_pipeline in filter_config:
                    filter_name = filter_config["name"]
                    filter_functions = filter_config["filter"]
                    components = []
                    for function in filter_functions:
                        kwargs = {key: function[key] for key in function if key != "function"}
                        components.append([function["function"], kwargs])
                    filter_pipeline = build_filter_ensemble(filter_name, components)
                self._filters.append(filter_pipeline)
        else:
            self._filters = [build_filter_ensemble("none", [["take_first", None]])]
        if self.config.fewshot_config is not None:
            self.sampler = samplers.get_sampler(self.config.fewshot_config.get("sampler", "default") if self.config.fewshot_config else "default")(list(self.fewshot_docs()), self, rnd=random.Random(1234))

        if self.has_test_docs():
            self.task_docs = self.test_docs()
        elif self.has_validation_docs():
            self.task_docs = self.validation_docs()
        else:
            assert False, f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!"

        # Test One Doc
        self.features = list(self.task_docs.features.keys())
        self.multiple_input = 0
        self.multiple_target = 0
        test_doc = self.task_docs[0]
        test_text = self.doc_to_text(test_doc)
        test_target = self.doc_to_target(test_doc)

        if self.config.doc_to_choice is not None:
            test_choice = self.doc_to_choice(test_doc)
            if type(test_choice) is not list:
                eval_logger.error("doc_to_choice must return list")
            else:
                num_choice = len(test_choice)

            if type(test_text) is int:
                self.multiple_input = num_choice
        else:
            test_choice = None

        if type(test_target) is list:
            self.multiple_target = len(test_target)
        else:
            if (type(test_target) is int) and (test_choice is not None):
                test_target = test_choice[test_target]
            else:
                test_target = str(test_target)

        if test_choice is not None:
            check_choices = test_choice
        else:
            check_choices = [test_target]
        if self.config.doc_to_choice is not None:
            for choice in check_choices:
                choice_has_whitespace = True if choice[0].isspace() else False
                delimiter_has_whitespace = True if self.config.target_delimiter.rstrip() != self.config.target_delimiter else False

                if delimiter_has_whitespace and choice_has_whitespace:
                    eval_logger.warning(f'Both target_delimiter and target choice: "{choice}" have whitespace')
                elif (not delimiter_has_whitespace) and (not choice_has_whitespace):
                    eval_logger.warning(f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" do not have whitespace, ignore if the language you are evaluating on does not require/use whitespace')