in src/alpaca_eval/main.py [0:0]
def get_completions(configs, df: pd.DataFrame, old_output_path: Optional[Path] = None):
columns_to_keep = ["dataset", "instruction", "output", "generator"]
columns_to_keep = [c for c in columns_to_keep if c in df.columns]
curr_outputs = df[columns_to_keep].copy()
is_loading_old_outputs = old_output_path is not None and old_output_path.exists()
assert len(configs) == 1
generator = list(configs.keys())[0]
configs = list(configs.values())[0]
if is_loading_old_outputs:
logging.info(f"Loading outputs from {old_output_path}")
old_outputs = utils.load_or_convert_to_dataframe(old_output_path)
# select only rows in curr_outputs that have "instruction" that are not in old_outputs
idx_found_old_outputs = curr_outputs["instruction"].isin(old_outputs["instruction"])
curr_outputs = curr_outputs[~idx_found_old_outputs]
assert (old_outputs["generator"] == generator).all()
logging.info(f"Found {len(old_outputs)}. Only generating {len(curr_outputs)} .")
if max_instances is not None:
curr_outputs = curr_outputs.iloc[:max_instances]
if len(curr_outputs) > 0:
prompts, _ = utils.make_prompts(
curr_outputs,
template=utils.read_or_return(constants.MODELS_CONFIG_DIR / configs["prompt_template"]),
)
fn_completions = decoders.get_fn_completions(configs["fn_completions"])
completions = fn_completions(prompts=prompts, **configs["completions_kwargs"])["completions"]
if is_strip_output:
completions = [c.strip() for c in completions]
curr_outputs["output"] = completions
curr_outputs["generator"] = generator
if is_loading_old_outputs:
curr_outputs = pd.concat([old_outputs, curr_outputs], axis=0)
return curr_outputs