lm_eval/tasks/bbh/fewshot/reasoning_about_colored_objects.yaml [9:118]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
fewshot_config:
  sampler: first_n
  samples:
  - input: 'On the nightstand, there is a red pencil, a purple mug, a burgundy keychain,
      a fuchsia teddy bear, a black plate, and a blue stress ball. What color is the
      stress ball?

      Options:

      (A) red

      (B) orange

      (C) yellow

      (D) green

      (E) blue

      (F) brown

      (G) magenta

      (H) fuchsia

      (I) mauve

      (J) teal

      (K) turquoise

      (L) burgundy

      (M) silver

      (N) gold

      (O) black

      (P) grey

      (Q) purple

      (R) pink'
    target: (E)
  - input: 'On the table, you see a bunch of objects arranged in a row: a purple paperclip,
      a pink stress ball, a brown keychain, a green scrunchiephone charger, a mauve
      fidget spinner, and a burgundy pen. What is the color of the object directly
      to the right of the stress ball?

      Options:

      (A) red

      (B) orange

      (C) yellow

      (D) green

      (E) blue

      (F) brown

      (G) magenta

      (H) fuchsia

      (I) mauve

      (J) teal

      (K) turquoise

      (L) burgundy

      (M) silver

      (N) gold

      (O) black

      (P) grey

      (Q) purple

      (R) pink'
    target: (F)
  - input: 'On the nightstand, you see the following items arranged in a row: a teal
      plate, a burgundy keychain, a yellow scrunchiephone charger, an orange mug,
      a pink notebook, and a grey cup. How many non-orange items do you see to the
      left of the teal item?

      Options:

      (A) zero

      (B) one

      (C) two

      (D) three

      (E) four

      (F) five

      (G) six'
    target: (A)
include: _fewshot_template_yaml
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



lm_eval/tasks/leaderboard/bbh_mc/reasoning_about_colored_objects.yaml [6:115]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
fewshot_config:
  sampler: first_n
  samples:
  - input: 'On the nightstand, there is a red pencil, a purple mug, a burgundy keychain,
      a fuchsia teddy bear, a black plate, and a blue stress ball. What color is the
      stress ball?

      Options:

      (A) red

      (B) orange

      (C) yellow

      (D) green

      (E) blue

      (F) brown

      (G) magenta

      (H) fuchsia

      (I) mauve

      (J) teal

      (K) turquoise

      (L) burgundy

      (M) silver

      (N) gold

      (O) black

      (P) grey

      (Q) purple

      (R) pink'
    target: (E)
  - input: 'On the table, you see a bunch of objects arranged in a row: a purple paperclip,
      a pink stress ball, a brown keychain, a green scrunchiephone charger, a mauve
      fidget spinner, and a burgundy pen. What is the color of the object directly
      to the right of the stress ball?

      Options:

      (A) red

      (B) orange

      (C) yellow

      (D) green

      (E) blue

      (F) brown

      (G) magenta

      (H) fuchsia

      (I) mauve

      (J) teal

      (K) turquoise

      (L) burgundy

      (M) silver

      (N) gold

      (O) black

      (P) grey

      (Q) purple

      (R) pink'
    target: (F)
  - input: 'On the nightstand, you see the following items arranged in a row: a teal
      plate, a burgundy keychain, a yellow scrunchiephone charger, an orange mug,
      a pink notebook, and a grey cup. How many non-orange items do you see to the
      left of the teal item?

      Options:

      (A) zero

      (B) one

      (C) two

      (D) three

      (E) four

      (F) five

      (G) six'
    target: (A)
include: _fewshot_template_yaml
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



