def _cnndm_iter()

in summarize_from_feedback/datasets/cnndm.py [0:0]

• 21 lines of code
• 4 McCabe index (conditional complexity)

def _cnndm_iter(split, subset="all", refs_with_bullets=False, clean_highlights=True):
    if split == "valid":
        split = "val"
    with blobs.open_file_cached(
        f"https://openaipublic.blob.core.windows.net/summarize-from-feedback/datasets/cnndm/url_lists/{subset}_{split}.txt"
    ) as f:
        urls = [line.strip() for line in f]

    with blobs.open_file_cached("https://openaipublic.blob.core.windows.net/summarize-from-feedback/datasets/cnndm/titles.json") as f:
        titles = json.load(f)
    urls_dir = blobs.download_directory_cached(
        f"https://openaipublic.blob.core.windows.net/summarize-from-feedback/datasets/cnndm/cache_{split}"
    )

    for url in urls:
        article_id, site = get_url_info(url)
        path = os.path.join(urls_dir, site, "stories", f"{article_id}.story")
        article, ref_sents = get_article_and_highlights(
            path, refs_with_bullets=refs_with_bullets, clean_highlights=clean_highlights
        )
        yield dict(
            article=article, reference=ref_sents, id=article_id, site=site, title=titles[article_id]
        )