def get_article_and_highlights()

in summarize_from_feedback/datasets/cnndm.py [0:0]

• 30 lines of code
• 9 McCabe index (conditional complexity)

def get_article_and_highlights(story_file, refs_with_bullets=False, clean_highlights=True):
    with open(story_file) as f:
        original_text = f.read()
    original_text = ftfy.fix_text(original_text)
    original_text = original_text.replace("\xa0", " ")  # hmm, not handled by ftfy?

    article_lines = []
    highlights = []
    has_seen_highlight = False

    for line in original_text.split("\n"):
        if line == "":
            continue  # empty line
        elif line.startswith("@highlight"):
            assert line == "@highlight"
            has_seen_highlight = True
        elif has_seen_highlight:
            if clean_highlights:
                line = clean_up_highlights(line)
            if not refs_with_bullets:
                line = fix_missing_period(line)
            if line:
                highlights.append(line)
        else:
            article_lines.append(line)

    article = "\n\n".join(article_lines)
    article = clean_up_start(article)

    if refs_with_bullets:
        highlights = "- " + "\n- ".join(highlights)
    else:
        highlights = " ".join(highlights)

    return article, highlights