evals/registry/solvers/make-me-pay.yaml

make-me-pay/baseline/gpt-3.5-turbo-16k:
  class: evals.elsuite.make_me_pay.solvers.lm_con_artist_solver:LMConArtistSolver
  args:
    base_solver_class: evals.solvers.providers.openai.openai_solver:OpenAISolver
    base_solver_args:
      completion_fn_options:
        model: gpt-3.5-turbo-16k
        extra_options:
          temperature: 1.0
          max_tokens: 1000

make-me-pay/baseline/gpt-4:
  class: evals.elsuite.make_me_pay.solvers.lm_con_artist_solver:LMConArtistSolver
  args:
    base_solver_class: evals.solvers.providers.openai.openai_solver:OpenAISolver
    base_solver_args:
      completion_fn_options:
        model: gpt-4
        extra_options:
          temperature: 1.0
          max_tokens: 1000

make-me-pay/baseline/gpt-4-32k:
  class: evals.elsuite.make_me_pay.solvers.lm_con_artist_solver:LMConArtistSolver
  args:
    base_solver_class: evals.solvers.providers.openai.openai_solver:OpenAISolver
    base_solver_args:
      completion_fn_options:
        model: gpt-4-32k
        extra_options:
          temperature: 1.0
          max_tokens: 1000

make-me-pay/gpt-3.5-turbo-16k:
  class: evals.elsuite.make_me_pay.solvers.lm_con_artist_solver:LMConArtistSolver
  args:
    base_solver_class: evals.solvers.nested.cot_solver:CoTSolver
    base_solver_args:
      cot_solver:
        class: evals.solvers.providers.openai.openai_solver:OpenAISolver
        args:
          completion_fn_options:
            model: gpt-3.5-turbo-16k
            extra_options:
              temperature: 1
              max_tokens: 1000
      extract_solver:
        class: evals.solvers.providers.openai.openai_solver:OpenAISolver
        args:
          completion_fn_options:
            model: gpt-3.5-turbo-16k
            extra_options:
              temperature: 1
              max_tokens: 1000
    lm_system_prompt: True
    time_reminder: True

make-me-pay/gpt-4:
  class: evals.elsuite.make_me_pay.solvers.lm_con_artist_solver:LMConArtistSolver
  args:
    base_solver_class: evals.solvers.nested.cot_solver:CoTSolver
    base_solver_args:
      cot_solver:
        class: evals.solvers.providers.openai.openai_solver:OpenAISolver
        args:
          completion_fn_options:
            model: gpt-4
            extra_options:
              temperature: 1
              max_tokens: 1000
      extract_solver:
        class: evals.solvers.providers.openai.openai_solver:OpenAISolver
        args:
          completion_fn_options:
            model: gpt-4
            extra_options:
              temperature: 1
              max_tokens: 1000
    lm_system_prompt: True
    time_reminder: True

make-me-pay/gpt-4-32k:
  class: evals.elsuite.make_me_pay.solvers.lm_con_artist_solver:LMConArtistSolver
  args:
    base_solver_class: evals.solvers.nested.cot_solver:CoTSolver
    base_solver_args:
      cot_solver:
        class: evals.solvers.providers.openai.openai_solver:OpenAISolver
        args:
          completion_fn_options:
            model: gpt-4-32k
            extra_options:
              temperature: 1
              max_tokens: 1000
      extract_solver:
        class: evals.solvers.providers.openai.openai_solver:OpenAISolver
        args:
          completion_fn_options:
            model: gpt-4-32k
            extra_options:
              temperature: 1
              max_tokens: 1000
    lm_system_prompt: True
    time_reminder: True

make-me-pay/cli:
  class: evals.solvers.human_cli_solver:HumanCliSolver