in tensorflow_datasets/text/c4_utils.py [0:0]
def clean_page(url_and_features,
citation_regex,
counter_inc_fn=None,
min_words_per_line=_MIN_WORDS_PER_LINE,
min_num_sentences=_MIN_NUM_SENTENCES,
max_word_length=_MAX_WORD_LENGTH):
"""Cleans a CommonCrawl page, yielding nothing if it should be skipped.
Cleaning removes lines with no end marks or with too few words. After line
filtering, pages are filtered out if they have too few sentences based on a
simple count of end marks.
Args:
url_and_features: tuple(string, dict), the url and features of the page.
citation_regex: Regex to use for finding Wikipedia-like citations to filter.
counter_inc_fn: function, a function taking the name of a counter to be
incremented and the (optional) amount. Defaults to a beam Metric counter.
min_words_per_line: int, the minimum number of words a line needs to not be
removed.
min_num_sentences: int, the minimum number of sentences a page needs to not
be skipped.
max_word_length: int, the maximum number of characters allowed in a word.
Lines containing a word with too many characters are removed.
Yields:
The url and cleaned text for the page.
"""
url, features = url_and_features
text = features["text"]
if not counter_inc_fn:
counter_inc_fn = get_counter_inc_fn("clean-page")
lines = text.splitlines()
valid_lines = []
num_sentences = 0
def line_has_too_long_word(line):
for word in line.split():
if len(word) > max_word_length:
return True
return False
for line in lines:
line = line.strip()
if line_has_too_long_word(line):
counter_inc_fn("line-filtered:too_long_word")
continue
line = citation_regex.sub("", line)
if not line.endswith(_END_MARKS) or line.endswith(_ELLIPSIS):
counter_inc_fn("line-filtered:no_endmark")
continue
if len(line.split()) < min_words_per_line:
counter_inc_fn("line-filtered:too_short")
continue
line_lower = line.lower()
# Remove documents which contain lorem ipsum
if "lorem ipsum" in line_lower:
counter_inc_fn("filtered:loremipsum")
return
# Remove "javascript must be enabled" notices
if "javascript" in line_lower:
counter_inc_fn("line-filtered:javascript")
continue
# Remove docs which probably contain javascript code
if "{" in line:
counter_inc_fn("filtered:squigglybracket")
return
# Remove policy lines
if any(p in line_lower for p in _POLICY_SUBSTRINGS):
counter_inc_fn("line-filtered:policy")
continue
num_sentences += len(_get_sentences(line))
valid_lines.append(line)
counter_inc_fn("line-passed")
if num_sentences < min_num_sentences:
counter_inc_fn("filtered:too_few_sentences")
return
counter_inc_fn("passed")
features["text"] = "\n".join(valid_lines).strip()
yield url, features