def nli_natural_prompt()

in src/lighteval/tasks/templates/nli.py [0:0]

• 41 lines of code
• 4 McCabe index (conditional complexity)

    def nli_natural_prompt(line: dict, task_name: str):
        labels = [capitalize(get_relation_label(label, translation_literals)) for label in relations]
        input_data = adapter_fn(line)
        if input_data is None:
            return None

        premise, hypothesis, label = input_data["premise"], input_data["hypothesis"], input_data["gold_idx"]

        premise = capitalize(input_data["premise"].rstrip(PUNCT))
        hypothesis = decapitalize(input_data["hypothesis"].rstrip(PUNCT))

        query = NLI_TEMPLATE_QUERY_CF.format(
            instruction=input_data.get("instruction", ""),
            premise=premise,
            full_stop=translation_literals.full_stop,
            confirmation_word=translation_literals.confirmation_word,
            question_mark=translation_literals.question_mark,
            word_space=translation_literals.word_space,
        )

        choices = [
            NLI_TEMPLATE_CONT_CF.format(
                hypothesis=hypothesis,
                label=label,
                comma=translation_literals.comma,
                word_space=translation_literals.word_space,
                sentence_space=translation_literals.sentence_space,
            )
            for label in labels
        ]

        unconditioned_query = NLI_TEMPLATE_QUERY_CF.format(
            instruction="",
            premise="",
            confirmation_word=translation_literals.confirmation_word,
            question_mark=translation_literals.question_mark,
            word_space="",
            full_stop=translation_literals.full_stop,
        )

        return Doc(
            task_name=task_name,
            query=query,
            choices=choices,
            gold_index=label,
            unconditioned_query=unconditioned_query,
        )