lm_eval/tasks/unscramble/cycle_letters.yaml

tag:
  - unscramble
task: cycle_letters
dataset_path: EleutherAI/unscramble
dataset_name: cycle_letters_in_word
output_type: generate_until
test_split: validation
doc_to_text: "{{context}}"
doc_to_target: "{{completion}}"
generation_kwargs:
  until:
    - "\n"
metric_list:
  - metric: exact_match
    aggregation: mean
    higher_is_better: true
    ignore_case: false
    ignore_punctuation: false
metadata:
  version: 2.0
dataset_kwargs:
  trust_remote_code: true