lm_eval/tasks/bbh/fewshot/penguins_in_a_table.yaml

dataset_name: penguins_in_a_table
description: 'Answer questions about a table of penguins and their attributes.


  '
doc_to_text: 'Q: {{input}}

  A:'
fewshot_config:
  sampler: first_n
  samples:
  - input: 'Here is a table where the first line is a header and each subsequent line
      is a penguin:  name, age, height (cm), weight (kg) Louis, 7, 50, 11 Bernard,
      5, 80, 13 Vincent, 9, 60, 11 Gwen, 8, 70, 15  For example: the age of Louis
      is 7, the weight of Gwen is 15 kg, the height of Bernard is 80 cm.  We now add
      a penguin to the table:

      James, 12, 90, 12

      How many penguins are less than 8 years old?

      Options:

      (A) 1

      (B) 2

      (C) 3

      (D) 4

      (E) 5'
    target: (B)
  - input: 'Here is a table where the first line is a header and each subsequent line
      is a penguin:  name, age, height (cm), weight (kg) Louis, 7, 50, 11 Bernard,
      5, 80, 13 Vincent, 9, 60, 11 Gwen, 8, 70, 15  For example: the age of Louis
      is 7, the weight of Gwen is 15 kg, the height of Bernard is 80 cm.  Which is
      the youngest penguin?

      Options:

      (A) Louis

      (B) Bernard

      (C) Vincent

      (D) Gwen

      (E) James'
    target: (B)
  - input: 'Here is a table where the first line is a header and each subsequent line
      is a penguin:  name, age, height (cm), weight (kg) Louis, 7, 50, 11 Bernard,
      5, 80, 13 Vincent, 9, 60, 11 Gwen, 8, 70, 15  For example: the age of Louis
      is 7, the weight of Gwen is 15 kg, the height of Bernard is 80 cm.  What is
      the name of the second penguin sorted by alphabetic order?

      Options:

      (A) Louis

      (B) Bernard

      (C) Vincent

      (D) Gwen

      (E) James'
    target: (D)
include: _fewshot_template_yaml
task: bbh_fewshot_penguins_in_a_table