def _few_shot_messages()

in evals/elsuite/function_deduction/solvers.py [0:0]

• 49 lines of code
• 2 McCabe index (conditional complexity)

    def _few_shot_messages(self, current_state) -> list[Message]:
        #   This is a bit hackish, but this way we can use self.cot_template (defined on CustomCoT),
        #   -> we'll have exactly the same system prompts in few shot part as we'd have in the eval
        cs = CurrentState(current_state.n_rounds, current_state.mode, current_state.test_inputs)
        task_state = TaskState("", [], cs)

        role = "system"

        messages = [
            (role, "I have a hidden function. What is your first action?"),
        ]
        messages.append((role, self.cot_template(task_state)))
        messages.append(
            (
                "assistant",
                (
                    "I don't know anything about the function yet. I should start with gathering some information, "
                    "e.g. by asking about the return value for 40."
                ),
            )
        )
        messages.append((role, self.extract_template(task_state)))
        messages.append(("assistant", "40"))
        messages.append((role, "calc(40) = 160"))
        cs.ask_update(40, 160)

        messages.append((role, self.cot_template(task_state)))
        messages.append(
            (
                "assistant",
                "Perhaps this function does multiplication by 4? That's not certain yet, so let's try some other number, e.g. 52.",
            )
        )
        messages.append((role, self.extract_template(task_state)))
        messages.append(("assistant", "52"))
        messages.append((role, "calc(52) = 204"))
        cs.ask_update(52, 204)

        messages.append((role, self.cot_template(task_state)))
        messages.append(
            (
                "assistant",
                (
                    "Now we have two results where the output is the input times 4. It seems that the function multiplies by 4. "
                    "I will make the guess now. 17 * 4 = 68, 36 * 4 = 144 and 68 * 4 = 272, so my guess will be 68 144 272."
                ),
            )
        )
        messages.append((role, self.extract_template(task_state)))
        messages.append(("assistant", "68 144 272"))
        messages.append((role, "Correct guess!"))
        messages.append(
            (role, "I now have a new function. Forget about the previous one, we start again.")
        )

        return [Message(*row) for row in messages]