def get_context()

in lm_eval/api/samplers.py [0:0]

• 27 lines of code
• 12 McCabe index (conditional complexity)

    def get_context(self, doc, num_fewshot):
        # draw an extra fewshot sample if using same split as evaluating on
        n_samples = (
            num_fewshot + 1
            if self.config.fewshot_split == self.config.test_split
            else num_fewshot
        )

        # draw `n_samples` docs from fewshot_docs
        fewshotex = self.sample(n_samples)

        # get rid of the doc that's the one we're evaluating, if it's in the fewshot
        # TODO: should we just stop people from using fewshot from same split as evaluating?
        selected_docs = [x for x in fewshotex if x != doc][:num_fewshot]

        labeled_examples = ""
        for doc in selected_docs:
            doc_content = self.doc_to_text(doc)
            doc_target = self.doc_to_target(doc)
            labeled_examples += (
                doc_content
                if self.config.doc_to_choice is None or isinstance(doc_content, str)
                else self.doc_to_choice(doc)[doc_content]
            )
            labeled_examples += self.target_delimiter
            labeled_examples += (
                str(doc_target[0])
                if isinstance(doc_target, list)
                else str(doc_target)
                if self.config.doc_to_choice is None or isinstance(doc_target, str)
                else str(self.doc_to_choice(doc)[doc_target])
            )
            labeled_examples += self.fewshot_delimiter

        return labeled_examples