def tldr_filtered_generator()

in summarize_from_feedback/datasets/tldr.py [0:0]

• 7 lines of code
• 5 McCabe index (conditional complexity)

def tldr_filtered_generator(split):
    assert split in ["test", "train", "valid"]

    gcs_path = f"https://openaipublic.blob.core.windows.net/summarize-from-feedback/datasets/tldr_3_filtered/{split}.jsonl"
    with blobs.open_file_cached(gcs_path, "rb") as f:
        datas = [json.loads(l) for l in f.readlines()]

    for data in datas:
        yield dict(reference=data["summary"], **{k: v for (k, v) in data.items() if k != "summary"})