def load_dataset()

in genai-on-vertex-ai/gemini/model_upgrades/multiturn_chat/vertex_script/eval.py [0:0]

• 7 lines of code
• 3 McCabe index (conditional complexity)

def load_dataset(dataset_local_path: str) -> pd.DataFrame:
    '''Load conversation histories from local files to a Pandas DataFrame.'''
    with open(dataset_local_path, 'r') as file:
        data = [json.loads(line) for line in file if line.strip()]
    dataset = pd.DataFrame(data)
    dataset['history'] = dataset['chat_path'].apply(lambda chat_path: open(chat_path, 'r').read())
    return dataset[['history']]