lm_eval/tasks/fld/fld_logical_formula_default.yaml

group:
  - fld_logical_formula
task: fld_logical_formula_default
dataset_path: hitachi-nlp/FLD.v2
dataset_name: default
training_split: train
validation_split: validation
test_split: test
doc_to_text: "Based on the provided facts ($context$), either prove or disprove the hypothesis or state that it is unknown. The facts and the hypothesis are written in logical formulas as follows: capital letters such as \"{A}\", \"{B}\", \"{AB}\" are predicates, small letters such as \"{a}\", \"{b}\", \"{ab}\" are constants, \"&\" is logical conjunction, \"v\" is logical disjunction, \"¬\" is negation, \"->\" is implication, \"(x)\" is \"for all x\", and \"(Ex)\" is \"for some x\". $hypothesis$ = {{hypothesis_formula}} ; $context$ = {{context_formula}} ; $proof$ = "
doc_to_target: world_assump_label
metric_list:
  - metric: exact_match
    aggregation: mean
    higher_is_better: true
filter_list:
  - name: remove_whitespace
    filter:
      - function: remove_whitespace
      - function: take_first
metadata:
  version: 2.0