build_obelics/05_filtering_web_docs.py
obelics/callers/filter_web_documents.py

build_obelics/02_bis_extract_html_get_image_urls_new_rules.py
obelics/callers/extract_web_documents.py

build_obelics/01_download_warc.py
obelics/callers/download_warc.py

build_obelics/02_extract_html_get_image_urls.py
obelics/callers/extract_html.py

build_obelics/02_extract_html_get_image_urls.py
obelics/visualization/global_visualization.py

build_obelics/02_bis_extract_html_get_image_urls_new_rules.py
obelics/callers/extract_html.py

build_obelics/02_extract_html_get_image_urls.py
obelics/callers/download_warc.py

build_obelics/06_01_create_set_image_urls_in_webdocs.py
obelics/callers/download_warc.py

build_obelics/06_01_create_set_image_urls_in_webdocs.py
obelics/callers/extract_html.py

build_obelics/02_bis_extract_html_get_image_urls_new_rules.py
obelics/callers/download_warc.py

build_obelics/02_extract_html_get_image_urls.py
obelics/processors/dom_tree_simplificator.py

build_obelics/04_merge_web_docs_with_images.py
obelics/callers/extract_html.py

build_obelics/03_dl_images_create_dataset.py
obelics/callers/extract_web_documents.py

build_obelics/04_merge_web_docs_with_images.py
obelics/callers/download_warc.py

build_obelics/09_06_line_dedup.py
obelics/processors/web_document_line_deduplication.py

build_obelics/02_extract_html_get_image_urls.py
obelics/callers/extract_web_documents.py

build_obelics/05_filtering_web_docs.py
obelics/visualization/web_document_and_filtering_visualization.py

build_obelics/02_extract_html_get_image_urls.py
obelics/callers/filter_web_documents.py

build_obelics/05_filtering_web_docs.py
obelics/visualization/choose_filtering_parameters_web_documents_node_level.py

build_obelics/04_merge_web_docs_with_images.py
obelics/callers/extract_web_documents.py

build_obelics/03_dl_images_create_dataset.py
obelics/callers/download_warc.py

build_obelics/01_download_warc.py
obelics/callers/extract_html.py

build_obelics/03_dl_images_create_dataset.py
obelics/callers/extract_html.py

build_obelics/06_01_create_set_image_urls_in_webdocs.py
obelics/callers/extract_web_documents.py

build_obelics/05_filtering_web_docs.py
obelics/utils/__init__.py

build_obelics/05_filtering_web_docs.py
obelics/callers/download_warc.py

build_obelics/06_03_remove_image_duplicates.py
obelics/callers/download_warc.py

build_obelics/02_bis_extract_html_get_image_urls_new_rules.py
obelics/callers/filter_web_documents.py

build_obelics/06_01_create_set_image_urls_in_webdocs.py
obelics/callers/filter_web_documents.py

build_obelics/06_03_remove_image_duplicates.py
obelics/callers/extract_html.py

build_obelics/04_merge_web_docs_with_images.py
obelics/callers/filter_web_documents.py

build_obelics/03_dl_images_create_dataset.py
obelics/callers/filter_web_documents.py

build_obelics/05_filtering_web_docs.py
obelics/callers/extract_html.py

build_obelics/05_filtering_web_docs.py
obelics/callers/extract_web_documents.py

build_obelics/01_download_warc.py
obelics/callers/filter_web_documents.py

build_obelics/06_03_remove_image_duplicates.py
obelics/callers/filter_web_documents.py

build_obelics/06_03_remove_image_duplicates.py
obelics/callers/extract_web_documents.py

build_obelics/01_download_warc.py
obelics/callers/extract_web_documents.py

build_obelics/02_extract_html_get_image_urls.py
obelics/processors/web_document_extractor.py

build_obelics/08_02_urldedup.py
obelics/callers/download_warc.py

build_obelics/11_02_get_docs_to_remove_by_set_img_urls_dedup.py
obelics/processors/web_document_extractor.py

build_obelics/09_04_get_domain_to_duplicated_texts.py
obelics/callers/filter_web_documents.py

build_obelics/12_02_remove_opt_out_images.py
obelics/processors/web_document_line_deduplication.py

build_obelics/11_01_create_set_img_urls.py
obelics/callers/download_warc.py

build_obelics/07_01_nsfw_image_filtering.py
obelics/callers/download_warc.py

build_obelics/07_01_nsfw_image_filtering.py
obelics/callers/extract_html.py

build_obelics/09_06_line_dedup.py
obelics/callers/extract_web_documents.py

build_obelics/07_01_nsfw_image_filtering.py
obelics/callers/filter_web_documents.py

build_obelics/09_04_get_domain_to_duplicated_texts.py
obelics/callers/download_warc.py

build_obelics/02_bis_extract_html_get_image_urls_new_rules.py
obelics/processors/web_document_line_deduplication.py

build_obelics/11_03_set_img_urls_dedup.py
obelics/processors/web_document_extractor.py

build_obelics/10_final_cleaning.py
obelics/processors/web_document_extractor.py

build_obelics/10_final_cleaning.py
obelics/processors/web_document_line_deduplication.py

build_obelics/09_05_merge_domain_to_duplicated_texts_sharded.py
obelics/callers/filter_web_documents.py

build_obelics/11_02_get_docs_to_remove_by_set_img_urls_dedup.py
obelics/callers/download_warc.py

build_obelics/11_03_set_img_urls_dedup.py
obelics/callers/filter_web_documents.py

build_obelics/09_07_merge_web_docs_texts_only_and_rest.py
obelics/callers/download_warc.py

build_obelics/13_final_processing.py
obelics/callers/filter_web_documents.py

build_obelics/05_filtering_web_docs.py
obelics/processors/web_document_line_deduplication.py

build_obelics/12_02_remove_opt_out_images.py
obelics/callers/download_warc.py

build_obelics/01_download_warc.py
obelics/processors/web_document_line_deduplication.py

build_obelics/11_03_set_img_urls_dedup.py
obelics/callers/extract_web_documents.py

build_obelics/08_02_urldedup.py
obelics/processors/web_document_extractor.py

build_obelics/09_02_get_domain_to_positions.py
obelics/callers/extract_html.py

build_obelics/13_final_processing.py
obelics/callers/extract_web_documents.py

build_obelics/12_02_remove_opt_out_images.py
obelics/processors/web_document_extractor.py

build_obelics/09_06_line_dedup.py
obelics/callers/filter_web_documents.py

build_obelics/13_final_processing.py
obelics/processors/web_document_line_deduplication.py

build_obelics/13_final_processing.py
obelics/processors/web_document_extractor.py

build_obelics/09_04_get_domain_to_duplicated_texts.py
obelics/callers/extract_web_documents.py

build_obelics/11_03_set_img_urls_dedup.py
obelics/processors/web_document_line_deduplication.py

build_obelics/09_04_get_domain_to_duplicated_texts.py
obelics/processors/web_document_line_deduplication.py

build_obelics/01_download_warc.py
obelics/processors/web_document_extractor.py

build_obelics/11_02_get_docs_to_remove_by_set_img_urls_dedup.py
obelics/processors/web_document_line_deduplication.py

build_obelics/09_07_merge_web_docs_texts_only_and_rest.py
obelics/processors/web_document_extractor.py

build_obelics/07_03_nsfw_image_removal.py
obelics/processors/web_document_line_deduplication.py

build_obelics/07_03_nsfw_image_removal.py
obelics/callers/extract_web_documents.py

build_obelics/09_07_merge_web_docs_texts_only_and_rest.py
obelics/processors/web_document_line_deduplication.py

build_obelics/12_02_remove_opt_out_images.py
obelics/callers/extract_html.py

build_obelics/12_02_remove_opt_out_images.py
obelics/callers/extract_web_documents.py

build_obelics/07_01_nsfw_image_filtering.py
obelics/processors/web_document_line_deduplication.py

build_obelics/11_02_get_docs_to_remove_by_set_img_urls_dedup.py
obelics/callers/extract_html.py

build_obelics/09_02_get_domain_to_positions.py
obelics/processors/web_document_extractor.py

build_obelics/09_07_merge_web_docs_texts_only_and_rest.py
obelics/callers/extract_web_documents.py

build_obelics/09_05_merge_domain_to_duplicated_texts_sharded.py
obelics/callers/download_warc.py

build_obelics/09_04_get_domain_to_duplicated_texts.py
obelics/callers/extract_html.py

build_obelics/09_06_line_dedup.py
obelics/callers/download_warc.py

build_obelics/09_06_line_dedup.py
obelics/processors/web_document_extractor.py

build_obelics/09_07_merge_web_docs_texts_only_and_rest.py
obelics/callers/filter_web_documents.py

build_obelics/13_final_processing.py
obelics/callers/download_warc.py

build_obelics/07_03_nsfw_image_removal.py
obelics/processors/web_document_extractor.py

build_obelics/09_07_merge_web_docs_texts_only_and_rest.py
obelics/callers/extract_html.py

build_obelics/04_merge_web_docs_with_images.py
obelics/processors/web_document_line_deduplication.py

build_obelics/09_01_create_web_docs_texts_only.py
obelics/callers/download_warc.py

build_obelics/08_02_urldedup.py
obelics/callers/filter_web_documents.py

build_obelics/09_02_get_domain_to_positions.py
obelics/processors/web_document_line_deduplication.py

build_obelics/03_dl_images_create_dataset.py
obelics/processors/web_document_line_deduplication.py

build_obelics/09_02_get_domain_to_positions.py
obelics/callers/extract_web_documents.py

build_obelics/10_final_cleaning.py
obelics/callers/extract_html.py

build_obelics/07_03_nsfw_image_removal.py
obelics/callers/filter_web_documents.py

build_obelics/06_01_create_set_image_urls_in_webdocs.py
obelics/processors/web_document_line_deduplication.py

build_obelics/12_02_remove_opt_out_images.py
obelics/callers/filter_web_documents.py

build_obelics/09_05_merge_domain_to_duplicated_texts_sharded.py
obelics/processors/web_document_extractor.py

build_obelics/10_final_cleaning.py
obelics/callers/download_warc.py

build_obelics/07_03_nsfw_image_removal.py
obelics/callers/download_warc.py

build_obelics/08_02_urldedup.py
obelics/processors/web_document_line_deduplication.py

build_obelics/11_02_get_docs_to_remove_by_set_img_urls_dedup.py
obelics/callers/filter_web_documents.py

build_obelics/02_bis_extract_html_get_image_urls_new_rules.py
obelics/processors/web_document_extractor.py

build_obelics/09_01_create_web_docs_texts_only.py
obelics/processors/web_document_extractor.py

build_obelics/11_01_create_set_img_urls.py
obelics/callers/filter_web_documents.py

build_obelics/09_02_get_domain_to_positions.py
obelics/callers/download_warc.py

build_obelics/08_02_urldedup.py
obelics/callers/extract_html.py

build_obelics/02_extract_html_get_image_urls.py
obelics/processors/web_document_line_deduplication.py

build_obelics/09_05_merge_domain_to_duplicated_texts_sharded.py
obelics/callers/extract_web_documents.py

build_obelics/11_03_set_img_urls_dedup.py
obelics/callers/download_warc.py

build_obelics/11_01_create_set_img_urls.py
obelics/callers/extract_web_documents.py

build_obelics/09_06_line_dedup.py
obelics/callers/extract_html.py

build_obelics/05_filtering_web_docs.py
obelics/processors/web_document_extractor.py

build_obelics/11_01_create_set_img_urls.py
obelics/processors/web_document_line_deduplication.py

build_obelics/10_final_cleaning.py
obelics/callers/extract_web_documents.py

build_obelics/09_05_merge_domain_to_duplicated_texts_sharded.py
obelics/processors/web_document_line_deduplication.py

build_obelics/08_02_urldedup.py
obelics/callers/extract_web_documents.py

build_obelics/09_02_get_domain_to_positions.py
obelics/callers/filter_web_documents.py

build_obelics/07_01_nsfw_image_filtering.py
obelics/callers/extract_web_documents.py

build_obelics/09_04_get_domain_to_duplicated_texts.py
obelics/processors/web_document_extractor.py

build_obelics/06_03_remove_image_duplicates.py
obelics/processors/web_document_line_deduplication.py

build_obelics/11_01_create_set_img_urls.py
obelics/callers/extract_html.py

build_obelics/09_01_create_web_docs_texts_only.py
obelics/callers/extract_html.py

build_obelics/11_03_set_img_urls_dedup.py
obelics/callers/extract_html.py

build_obelics/10_final_cleaning.py
obelics/callers/filter_web_documents.py

build_obelics/11_02_get_docs_to_remove_by_set_img_urls_dedup.py
obelics/callers/extract_web_documents.py

build_obelics/07_01_nsfw_image_filtering.py
obelics/processors/web_document_extractor.py

build_obelics/09_01_create_web_docs_texts_only.py
obelics/callers/filter_web_documents.py

build_obelics/09_01_create_web_docs_texts_only.py
obelics/processors/web_document_line_deduplication.py

build_obelics/07_03_nsfw_image_removal.py
obelics/callers/extract_html.py

build_obelics/06_03_remove_image_duplicates.py
obelics/processors/web_document_extractor.py

build_obelics/09_01_create_web_docs_texts_only.py
obelics/callers/extract_web_documents.py

build_obelics/04_merge_web_docs_with_images.py
obelics/processors/web_document_extractor.py

build_obelics/09_05_merge_domain_to_duplicated_texts_sharded.py
obelics/callers/extract_html.py

build_obelics/06_01_create_set_image_urls_in_webdocs.py
obelics/processors/web_document_extractor.py

build_obelics/03_dl_images_create_dataset.py
obelics/processors/web_document_extractor.py

build_obelics/13_final_processing.py
obelics/callers/extract_html.py

build_obelics/11_01_create_set_img_urls.py
obelics/processors/web_document_extractor.py