def load_dataset()

in genai-on-vertex-ai/gemini/model_upgrades/summarization/vertex_script/eval.py [0:0]

• 6 lines of code
• 3 McCabe index (conditional complexity)

def load_dataset(dataset_local_path: str):
    with open(dataset_local_path, 'r') as file:
        data = [json.loads(line) for line in file if line.strip()]
    df = pd.DataFrame(data)
    df['document_text'] = df['document_path'].apply(lambda doc_path: open(doc_path, 'r').read())
    return df[['document_text', 'reference']]