Path	Lines of Code
catalog-info.yaml	59
lib/constants.rb	29
lib/crawler.rb	12
lib/crawler/api/config.rb	341
lib/crawler/api/crawl.rb	185
lib/crawler/cli.rb	11
lib/crawler/cli/crawl.rb	16
lib/crawler/cli/helpers.rb	43
lib/crawler/cli/schedule.rb	34
lib/crawler/cli/urltest.rb	17
lib/crawler/cli/validate.rb	31
lib/crawler/cli/version.rb	12
lib/crawler/content_engine/extractor.rb	49
lib/crawler/content_engine/transformer.rb	38
lib/crawler/content_engine/utils.rb	67
lib/crawler/coordinator.rb	563
lib/crawler/core_ext.rb	5
lib/crawler/data/crawl_result/base.rb	69
lib/crawler/data/crawl_result/content_extractable_file.rb	26
lib/crawler/data/crawl_result/error.rb	23
lib/crawler/data/crawl_result/html.rb	153
lib/crawler/data/crawl_result/http_auth_disallowed_error.rb	20
lib/crawler/data/crawl_result/redirect.rb	37
lib/crawler/data/crawl_result/redirect_error.rb	16
lib/crawler/data/crawl_result/robots_txt.rb	10
lib/crawler/data/crawl_result/sitemap.rb	50
lib/crawler/data/crawl_result/success.rb	19
lib/crawler/data/crawl_result/unsupported_content_type.rb	21
lib/crawler/data/crawl_task.rb	61
lib/crawler/data/domain.rb	33
lib/crawler/data/extraction/rule.rb	98
lib/crawler/data/extraction/ruleset.rb	52
lib/crawler/data/extraction/url_filter.rb	37
lib/crawler/data/link.rb	60
lib/crawler/data/rule.rb	30
lib/crawler/data/rule_engine_outcome.rb	90
lib/crawler/data/seen_urls.rb	28
lib/crawler/data/url.rb	46
lib/crawler/data/url_queue.rb	19
lib/crawler/data/url_queue/base.rb	55
lib/crawler/data/url_queue/memory_only.rb	52
lib/crawler/document_mapper.rb	88
lib/crawler/event_generator.rb	279
lib/crawler/executor.rb	10
lib/crawler/http_client.rb	276
lib/crawler/http_executor.rb	286
lib/crawler/http_header_service.rb	73
lib/crawler/http_utils/all_trusting_trust_manager.rb	17
lib/crawler/http_utils/config.rb	104
lib/crawler/http_utils/exceptions.rb	150
lib/crawler/http_utils/filtering_dns_resolver.rb	51
lib/crawler/http_utils/response.rb	136
lib/crawler/logging/handler/base.rb	19
lib/crawler/logging/handler/file.rb	50
lib/crawler/logging/handler/stdout.rb	49
lib/crawler/logging/logger.rb	44
lib/crawler/mock_event_logger.rb	13
lib/crawler/mock_executor.rb	23
lib/crawler/output_sink.rb	14
lib/crawler/output_sink/base.rb	41
lib/crawler/output_sink/console.rb	17
lib/crawler/output_sink/elasticsearch.rb	222
lib/crawler/output_sink/file.rb	20
lib/crawler/output_sink/mock.rb	17
lib/crawler/output_sink/null.rb	9
lib/crawler/robots_txt_parser.rb	54
lib/crawler/robots_txt_service.rb	72
lib/crawler/rule_engine/base.rb	68
lib/crawler/stats.rb	67
lib/crawler/url_validator.rb	124
lib/crawler/url_validator/crawl_rules_check_concern.rb	17
lib/crawler/url_validator/dns_check_concern.rb	30
lib/crawler/url_validator/domain_access_check_concern.rb	12
lib/crawler/url_validator/domain_uniqueness_check_concern.rb	12
lib/crawler/url_validator/result.rb	19
lib/crawler/url_validator/robots_txt_check_concern.rb	50
lib/crawler/url_validator/tcp_check_concern.rb	24
lib/crawler/url_validator/url_check_concern.rb	18
lib/crawler/url_validator/url_content_check_concern.rb	57
lib/crawler/url_validator/url_request_check_concern.rb	122
lib/crawler/utils.rb	22
lib/environment.rb	9
lib/errors.rb	6
lib/es/bulk_queue.rb	70
lib/es/client.rb	198
script/licenses/generate_notice.rb	43
script/licenses/lib/third_party.rb	79
script/support/string_colors.rb	14
spec/factories/crawl_results.rb	104
spec/fixtures/sitemap/sitemap_index.xml	11
spec/fixtures/sitemap/sitemap_no_urls.xml	3
spec/fixtures/sitemap/sitemap_urlset.xml	27
spec/integration/charset_spec.rb	44
spec/integration/content_extraction_spec.rb	44
spec/integration/headers_spec.rb	75
spec/integration/legacy_sitemaps_spec.rb	69
spec/integration/nofollow_spec.rb	30
spec/integration/redirects_spec.rb	40
spec/integration/response_content_type_spec.rb	28
spec/integration/response_limits_spec.rb	34
spec/integration/robots_txt_spec.rb	169
spec/integration/seed_spec.rb	15
spec/integration/sitemap_spec.rb	36
spec/integration/sitemap_xxe_spec.rb	69
spec/integration/timeouts/request_timeout_spec.rb	65
spec/integration/timeouts/socket_timeout_spec.rb	23
spec/integration/url_fragments_spec.rb	22
spec/lib/crawler/api/config_spec.rb	218
spec/lib/crawler/api/crawl_spec.rb	183
spec/lib/crawler/cli/crawl_spec.rb	63
spec/lib/crawler/cli/helpers_spec.rb	196
spec/lib/crawler/cli/schedule_spec.rb	50
spec/lib/crawler/cli/urltest_spec.rb	49
spec/lib/crawler/cli/validate_spec.rb	53
spec/lib/crawler/cli/version_spec.rb	9
spec/lib/crawler/content_engine/extractor_spec.rb	188
spec/lib/crawler/content_engine/transformer_spec.rb	213
spec/lib/crawler/content_engine/utils_spec.rb	35
spec/lib/crawler/coordinator_spec.rb	708
spec/lib/crawler/data/crawl_result/html_spec.rb	365
spec/lib/crawler/data/crawl_result/sitemap_spec.rb	194
spec/lib/crawler/data/crawl_result_spec.rb	72
spec/lib/crawler/data/crawl_task_spec.rb	10
spec/lib/crawler/data/domain_spec.rb	31
spec/lib/crawler/data/extraction/rule_spec.rb	162
spec/lib/crawler/data/extraction/ruleset_spec.rb	61
spec/lib/crawler/data/extraction/url_filter_spec.rb	58
spec/lib/crawler/data/link_spec.rb	119
spec/lib/crawler/data/rule_spec.rb	22
spec/lib/crawler/data/url_queue/memory_only_spec.rb	54
spec/lib/crawler/data/url_queue_spec.rb	14
spec/lib/crawler/data/url_spec.rb	55
spec/lib/crawler/document_mapper_spec.rb	219
spec/lib/crawler/event_generator_spec.rb	150
spec/lib/crawler/http_client_spec.rb	318
spec/lib/crawler/http_executor_spec.rb	366
spec/lib/crawler/http_utils/bad_ssl_spec.rb	109
spec/lib/crawler/http_utils/config_spec.rb	18
spec/lib/crawler/http_utils/filtering_dns_resolver_spec.rb	55
spec/lib/crawler/http_utils/response_spec.rb	42
spec/lib/crawler/logging/crawllogger_spec.rb	68
spec/lib/crawler/output_sink/elasticsearch_spec.rb	594
spec/lib/crawler/output_sink/file_spec.rb	26
spec/lib/crawler/output_sink_spec.rb	36
spec/lib/crawler/robots_txt_parser_spec.rb	133
spec/lib/crawler/rule_engine/base_spec.rb	182
spec/lib/crawler/stats_spec.rb	85
spec/lib/crawler/url_validator/crawl_rules_check_spec.rb	49
spec/lib/crawler/url_validator/dns_check_spec.rb	62
spec/lib/crawler/url_validator/domain_access_check_spec.rb	33
spec/lib/crawler/url_validator/domain_uniqueness_check_spec.rb	33
spec/lib/crawler/url_validator/robots_txt_check_spec.rb	104
spec/lib/crawler/url_validator/tcp_check_spec.rb	86
spec/lib/crawler/url_validator/url_check_spec.rb	74
spec/lib/crawler/url_validator/url_content_check_spec.rb	148
spec/lib/crawler/url_validator/url_request_check_spec.rb	165
spec/lib/crawler/url_validator_spec.rb	132
spec/lib/crawler_spec.rb	16
spec/lib/environment_spec.rb	5
spec/lib/es/bulk_queue_spec.rb	124
spec/lib/es/client_spec.rb	480
spec/spec_helper.rb	38
spec/support/cli_helpers.rb	32
spec/support/crawl_response_matchers.rb	46
spec/support/faux/faux_crawl.rb	153
spec/support/faux/results_collection.rb	17
spec/support/fixtures.rb	17
spec/support/mock_response.rb	11