id: 1 unit: validate_url_request file: lib/crawler/url_validator/url_request_check_concern.rb start line: 12 end line: 99 size: 68 LOC McCabe index: 17 number of parameters: 0 id: 2 unit: check_discovered_url file: lib/crawler/coordinator.rb start line: 661 end line: 738 size: 66 LOC McCabe index: 14 number of parameters: 4 id: 3 unit: generate_crawl_result file: lib/crawler/http_executor.rb start line: 201 end line: 270 size: 55 LOC McCabe index: 7 number of parameters: 2 id: 4 unit: run file: lib/crawler/http_executor.rb start line: 40 end line: 102 size: 47 LOC McCabe index: 12 number of parameters: 2 id: 5 unit: validate_robots_txt file: lib/crawler/url_validator/robots_txt_check_concern.rb start line: 12 end line: 73 size: 45 LOC McCabe index: 10 number of parameters: 0 id: 6 unit: configure_crawl file: spec/support/faux/faux_crawl.rb start line: 151 end line: 202 size: 41 LOC McCabe index: 3 number of parameters: 0 id: 7 unit: add_urls_to_backlog file: lib/crawler/coordinator.rb start line: 574 end line: 622 size: 40 LOC McCabe index: 11 number of parameters: 6 id: 8 unit: redirect_validation_result file: lib/crawler/url_validator/url_request_check_concern.rb start line: 101 end line: 149 size: 36 LOC McCabe index: 9 number of parameters: 1 id: 9 unit: validate_url_content file: lib/crawler/url_validator/url_content_check_concern.rb start line: 12 end line: 54 size: 33 LOC McCabe index: 6 number of parameters: 0 id: 10 unit: process_crawl_result file: lib/crawler/coordinator.rb start line: 382 end line: 420 size: 30 LOC McCabe index: 5 number of parameters: 2 id: 11 unit: head file: lib/crawler/http_client.rb start line: 82 end line: 118 size: 30 LOC McCabe index: 7 number of parameters: 2 id: 12 unit: get file: lib/crawler/http_client.rb start line: 120 end line: 155 size: 30 LOC McCabe index: 7 number of parameters: 2 id: 13 unit: self.node_descendant_text file: lib/crawler/content_engine/utils.rb start line: 33 end line: 84 size: 30 LOC McCabe index: 14 number of parameters: 2 id: 14 unit: handling_http_errors file: lib/crawler/http_executor.rb start line: 104 end line: 134 size: 30 LOC McCabe index: 9 number of parameters: 1 id: 15 unit: crawl_finished? file: lib/crawler/coordinator.rb start line: 276 end line: 313 size: 29 LOC McCabe index: 7 number of parameters: 0 id: 16 unit: add_url_to_backlog file: lib/crawler/coordinator.rb start line: 626 end line: 657 size: 29 LOC McCabe index: 3 number of parameters: 6 id: 17 unit: failure_message_template file: spec/support/crawl_response_matchers.rb start line: 39 end line: 73 size: 27 LOC McCabe index: 4 number of parameters: 2 id: 18 unit: extract_and_enqueue_html_links file: lib/crawler/coordinator.rb start line: 446 end line: 480 size: 27 LOC McCabe index: 4 number of parameters: 2 id: 19 unit: output_crawl_result file: lib/crawler/coordinator.rb start line: 540 end line: 571 size: 27 LOC McCabe index: 8 number of parameters: 1 id: 20 unit: start! file: lib/crawler/api/crawl.rb start line: 86 end line: 115 size: 26 LOC McCabe index: 9 number of parameters: 0 id: 21 unit: url_fetch file: lib/crawler/event_generator.rb start line: 156 end line: 185 size: 26 LOC McCabe index: 4 number of parameters: 3 id: 22 unit: start_sites file: spec/support/faux/faux_crawl.rb start line: 94 end line: 128 size: 25 LOC McCabe index: 6 number of parameters: 0 id: 23 unit: extract_and_enqueue_sitemap_links file: lib/crawler/coordinator.rb start line: 482 end line: 509 size: 25 LOC McCabe index: 3 number of parameters: 2 id: 24 unit: load_robots_txt file: lib/crawler/coordinator.rb start line: 174 end line: 201 size: 24 LOC McCabe index: 8 number of parameters: 1 id: 25 unit: enqueue_sitemaps file: lib/crawler/coordinator.rb start line: 216 end line: 243 size: 24 LOC McCabe index: 4 number of parameters: 0 id: 26 unit: validate_dns file: lib/crawler/url_validator/dns_check_concern.rb start line: 14 end line: 43 size: 24 LOC McCabe index: 4 number of parameters: 0 id: 27 unit: raise_if_necessary file: lib/es/client.rb start line: 183 end line: 213 size: 24 LOC McCabe index: 5 number of parameters: 1 id: 28 unit: extract_links file: lib/crawler/coordinator.rb start line: 511 end line: 537 size: 23 LOC McCabe index: 5 number of parameters: 2 id: 29 unit: check_queue_size! file: lib/crawler/data/url_queue/memory_only.rb start line: 38 end line: 63 size: 23 LOC McCabe index: 4 number of parameters: 0 id: 30 unit: flush file: lib/crawler/output_sink/elasticsearch.rb start line: 158 end line: 183 size: 22 LOC McCabe index: 6 number of parameters: 0 id: 31 unit: url_output file: lib/crawler/event_generator.rb start line: 230 end line: 255 size: 22 LOC McCabe index: 1 number of parameters: 8 id: 32 unit: configure_ssl file: lib/es/client.rb start line: 148 end line: 176 size: 22 LOC McCabe index: 5 number of parameters: 1 id: 33 unit: generate_content_extractable_file_crawl_result file: lib/crawler/http_executor.rb start line: 330 end line: 351 size: 21 LOC McCabe index: 4 number of parameters: 3 id: 34 unit: consume_http_entity file: lib/crawler/http_utils/response.rb start line: 147 end line: 184 size: 21 LOC McCabe index: 6 number of parameters: 2 id: 35 unit: self.for_java_error file: lib/crawler/http_utils/exceptions.rb start line: 101 end line: 123 size: 21 LOC McCabe index: 7 number of parameters: 1 id: 36 unit: initialize file: spec/support/faux/faux_crawl.rb start line: 51 end line: 73 size: 20 LOC McCabe index: 3 number of parameters: 1 id: 37 unit: check_connection_pool_stats! file: lib/crawler/http_client.rb start line: 360 end line: 381 size: 20 LOC McCabe index: 3 number of parameters: 0 id: 38 unit: http_client file: lib/crawler/http_executor.rb start line: 137 end line: 157 size: 20 LOC McCabe index: 2 number of parameters: 0 id: 39 unit: handle_redirect file: lib/crawler/http_executor.rb start line: 179 end line: 200 size: 20 LOC McCabe index: 2 number of parameters: 3 id: 40 unit: fetch_purge_docs file: lib/crawler/output_sink/elasticsearch.rb start line: 103 end line: 124 size: 20 LOC McCabe index: 2 number of parameters: 1 id: 41 unit: execute_with_retry file: lib/es/client.rb start line: 228 end line: 248 size: 20 LOC McCabe index: 3 number of parameters: 1 id: 42 unit: initialize file: lib/crawler/coordinator.rb start line: 36 end line: 59 size: 19 LOC McCabe index: 1 number of parameters: 1 id: 43 unit: validate_selector file: lib/crawler/data/extraction/rule.rb start line: 104 end line: 124 size: 19 LOC McCabe index: 6 number of parameters: 0 id: 44 unit: validate_tcp file: lib/crawler/url_validator/tcp_check_concern.rb start line: 12 end line: 33 size: 19 LOC McCabe index: 4 number of parameters: 0 id: 45 unit: start_url_test! file: lib/crawler/api/crawl.rb start line: 117 end line: 137 size: 19 LOC McCabe index: 7 number of parameters: 1 id: 46 unit: self.traverse! file: lib/crawler/content_engine/transformer.rb start line: 30 end line: 52 size: 18 LOC McCabe index: 7 number of parameters: 2 id: 47 unit: purge file: lib/crawler/output_sink/elasticsearch.rb start line: 125 end line: 146 size: 18 LOC McCabe index: 2 number of parameters: 1 id: 48 unit: add file: lib/es/bulk_queue.rb start line: 42 end line: 64 size: 18 LOC McCabe index: 3 number of parameters: 2 id: 49 unit: connection_config file: lib/es/client.rb start line: 40 end line: 60 size: 18 LOC McCabe index: 1 number of parameters: 2 id: 50 unit: handle_request file: spec/integration/timeouts/request_timeout_spec.rb start line: 39 end line: 67 size: 17 LOC McCabe index: 2 number of parameters: 2 id: 51 unit: generate_html_crawl_result file: lib/crawler/http_executor.rb start line: 292 end line: 309 size: 17 LOC McCabe index: 2 number of parameters: 3 id: 52 unit: generate_xml_sitemap_crawl_result file: lib/crawler/http_executor.rb start line: 311 end line: 328 size: 17 LOC McCabe index: 2 number of parameters: 3 id: 53 unit: configure_ssl_ca_certificates! file: lib/crawler/api/config.rb start line: 269 end line: 288 size: 17 LOC McCabe index: 4 number of parameters: 0 id: 54 unit: setup_system_logger file: lib/crawler/api/config.rb start line: 424 end line: 443 size: 17 LOC McCabe index: 2 number of parameters: 3 id: 55 unit: write file: lib/crawler/output_sink/elasticsearch.rb start line: 80 end line: 102 size: 17 LOC McCabe index: 3 number of parameters: 1 id: 56 unit: authorization_header_for_url file: lib/crawler/http_header_service.rb start line: 68 end line: 88 size: 16 LOC McCabe index: 5 number of parameters: 1 id: 57 unit: log file: lib/crawler/logging/handler/file.rb start line: 29 end line: 45 size: 16 LOC McCabe index: 6 number of parameters: 2 id: 58 unit: log file: lib/crawler/logging/handler/stdout.rb start line: 27 end line: 43 size: 16 LOC McCabe index: 6 number of parameters: 2 id: 59 unit: crawl_start file: lib/crawler/event_generator.rb start line: 56 end line: 73 size: 16 LOC McCabe index: 2 number of parameters: 2 id: 60 unit: prepare_crawl_task file: lib/crawler/coordinator.rb start line: 332 end line: 352 size: 15 LOC McCabe index: 5 number of parameters: 0 id: 61 unit: base_url file: lib/crawler/data/crawl_result/html.rb start line: 37 end line: 56 size: 15 LOC McCabe index: 6 number of parameters: 0 id: 62 unit: validate_url_filter file: lib/crawler/data/extraction/url_filter.rb start line: 25 end line: 42 size: 15 LOC McCabe index: 7 number of parameters: 0 id: 63 unit: output_crawl_result_outcome file: lib/crawler/rule_engine/base.rb start line: 64 end line: 82 size: 15 LOC McCabe index: 8 number of parameters: 1 id: 64 unit: run_schedule file: lib/crawler/cli/schedule.rb start line: 30 end line: 50 size: 15 LOC McCabe index: 2 number of parameters: 2 id: 65 unit: initialize file: lib/crawler/api/config.rb start line: 218 end line: 244 size: 15 LOC McCabe index: 1 number of parameters: 1 id: 66 unit: configure_seed_urls! file: lib/crawler/api/config.rb start line: 352 end line: 372 size: 15 LOC McCabe index: 5 number of parameters: 0 id: 67 unit: print_crawl_ingestion_results file: lib/crawler/api/crawl.rb start line: 211 end line: 230 size: 15 LOC McCabe index: 4 number of parameters: 1 id: 68 unit: initialize file: lib/crawler/output_sink/elasticsearch.rb start line: 25 end line: 49 size: 15 LOC McCabe index: 4 number of parameters: 1 id: 69 unit: init_ingestion_stats file: lib/crawler/output_sink/elasticsearch.rb start line: 243 end line: 258 size: 15 LOC McCabe index: 1 number of parameters: 0 id: 70 unit: url_seed file: lib/crawler/event_generator.rb start line: 139 end line: 154 size: 15 LOC McCabe index: 1 number of parameters: 5 id: 71 unit: run_purge_crawl! file: lib/crawler/coordinator.rb start line: 100 end line: 120 size: 14 LOC McCabe index: 3 number of parameters: 0 id: 72 unit: run_urltest_crawl! file: lib/crawler/coordinator.rb start line: 121 end line: 135 size: 14 LOC McCabe index: 1 number of parameters: 1 id: 73 unit: proxy_host file: lib/crawler/http_client.rb start line: 317 end line: 333 size: 14 LOC McCabe index: 2 number of parameters: 0 id: 74 unit: self.limit_bytesize file: lib/crawler/content_engine/utils.rb start line: 92 end line: 107 size: 14 LOC McCabe index: 4 number of parameters: 2 id: 75 unit: initialize file: lib/crawler/data/link.rb start line: 17 end line: 33 size: 14 LOC McCabe index: 10 number of parameters: 3 id: 76 unit: extract_links file: lib/crawler/data/crawl_result/html.rb start line: 58 end line: 79 size: 14 LOC McCabe index: 5 number of parameters: 2 id: 77 unit: url_disallowed_outcome file: lib/crawler/robots_txt_service.rb start line: 42 end line: 59 size: 14 LOC McCabe index: 5 number of parameters: 1 id: 78 unit: url_components file: lib/crawler/document_mapper.rb start line: 88 end line: 102 size: 14 LOC McCabe index: 2 number of parameters: 1 id: 79 unit: body file: lib/crawler/http_utils/response.rb start line: 39 end line: 56 size: 14 LOC McCabe index: 4 number of parameters: 0 id: 80 unit: remove_private_addresses file: lib/crawler/http_utils/filtering_dns_resolver.rb start line: 40 end line: 58 size: 14 LOC McCabe index: 5 number of parameters: 2 id: 81 unit: status file: lib/crawler/api/crawl.rb start line: 142 end line: 156 size: 14 LOC McCabe index: 1 number of parameters: 0 id: 82 unit: url_extracted file: lib/crawler/event_generator.rb start line: 214 end line: 228 size: 14 LOC McCabe index: 1 number of parameters: 8 id: 83 unit: get_retry_configuration file: lib/es/client.rb start line: 114 end line: 134 size: 14 LOC McCabe index: 7 number of parameters: 1 id: 84 unit: run_crawl_loop file: lib/crawler/coordinator.rb start line: 314 end line: 329 size: 13 LOC McCabe index: 4 number of parameters: 0 id: 85 unit: new_http_client file: lib/crawler/http_client.rb start line: 179 end line: 192 size: 13 LOC McCabe index: 2 number of parameters: 0 id: 86 unit: ssl_trust_managers file: lib/crawler/http_client.rb start line: 221 end line: 243 size: 13 LOC McCabe index: 2 number of parameters: 0 id: 87 unit: self.execute_rule file: lib/crawler/content_engine/extractor.rb start line: 25 end line: 41 size: 13 LOC McCabe index: 3 number of parameters: 2 id: 88 unit: meta_tags_elastic file: lib/crawler/data/crawl_result/html.rb start line: 123 end line: 139 size: 13 LOC McCabe index: 2 number of parameters: 1 id: 89 unit: data_attributes_from_body file: lib/crawler/data/crawl_result/html.rb start line: 140 end line: 154 size: 13 LOC McCabe index: 2 number of parameters: 1 id: 90 unit: crawl_rules_outcome_by_policy file: lib/crawler/rule_engine/base.rb start line: 49 end line: 63 size: 13 LOC McCabe index: 3 number of parameters: 2 id: 91 unit: validate_url file: lib/crawler/url_validator/url_check_concern.rb start line: 12 end line: 25 size: 13 LOC McCabe index: 6 number of parameters: 0 id: 92 unit: unauthorized_validation_result file: lib/crawler/url_validator/url_request_check_concern.rb start line: 150 end line: 167 size: 13 LOC McCabe index: 4 number of parameters: 1 id: 93 unit: configure_extraction_rules! file: lib/crawler/api/config.rb start line: 394 end line: 410 size: 13 LOC McCabe index: 5 number of parameters: 0 id: 94 unit: setup_event_logger file: lib/crawler/api/config.rb start line: 444 end line: 457 size: 13 LOC McCabe index: 2 number of parameters: 3 id: 95 unit: initialize file: lib/crawler/api/crawl.rb start line: 22 end line: 45 size: 13 LOC McCabe index: 5 number of parameters: 1 id: 96 unit: print_url_test_results file: lib/crawler/api/crawl.rb start line: 194 end line: 210 size: 13 LOC McCabe index: 2 number of parameters: 1 id: 97 unit: verify_es_connection file: lib/crawler/output_sink/elasticsearch.rb start line: 50 end line: 64 size: 13 LOC McCabe index: 2 number of parameters: 0 id: 98 unit: url_reprocessed file: lib/crawler/event_generator.rb start line: 257 end line: 272 size: 13 LOC McCabe index: 1 number of parameters: 6 id: 99 unit: url_fields file: lib/crawler/event_generator.rb start line: 293 end line: 308 size: 13 LOC McCabe index: 1 number of parameters: 1 id: 100 unit: configure_host_port file: lib/es/client.rb start line: 96 end line: 113 size: 13 LOC McCabe index: 3 number of parameters: 1 id: 101 unit: update_from_event file: lib/crawler/stats.rb start line: 77 end line: 89 size: 12 LOC McCabe index: 5 number of parameters: 1 id: 102 unit: self.shutdown_on_finalize file: lib/crawler/http_client.rb start line: 162 end line: 174 size: 12 LOC McCabe index: 2 number of parameters: 2 id: 103 unit: self.path_pattern file: lib/crawler/utils.rb start line: 14 end line: 26 size: 12 LOC McCabe index: 5 number of parameters: 2 id: 104 unit: headings file: lib/crawler/data/crawl_result/html.rb start line: 187 end line: 201 size: 12 LOC McCabe index: 4 number of parameters: 1 id: 105 unit: initialize file: lib/crawler/data/crawl_result/unsupported_content_type.rb start line: 14 end line: 27 size: 12 LOC McCabe index: 2 number of parameters: 3 id: 106 unit: discover_url_outcome file: lib/crawler/rule_engine/base.rb start line: 21 end line: 37 size: 12 LOC McCabe index: 5 number of parameters: 1 id: 107 unit: validate_crawl_rules file: lib/crawler/url_validator/crawl_rules_check_concern.rb start line: 12 end line: 26 size: 12 LOC McCabe index: 3 number of parameters: 0 id: 108 unit: print_final_crawl_status file: lib/crawler/api/crawl.rb start line: 231 end line: 243 size: 12 LOC McCabe index: 1 number of parameters: 0 id: 109 unit: prefixed_ecs_event file: lib/crawler/event_generator.rb start line: 118 end line: 130 size: 12 LOC McCabe index: 2 number of parameters: 2 id: 110 unit: url_discover file: lib/crawler/event_generator.rb start line: 194 end line: 206 size: 12 LOC McCabe index: 1 number of parameters: 6 id: 111 unit: initialize file: lib/es/bulk_queue.rb start line: 19 end line: 33 size: 12 LOC McCabe index: 3 number of parameters: 3 id: 112 unit: paginated_search file: lib/es/client.rb start line: 73 end line: 87 size: 12 LOC McCabe index: 2 number of parameters: 2 id: 113 unit: configure_auth file: lib/es/client.rb start line: 135 end line: 147 size: 12 LOC McCabe index: 4 number of parameters: 1 id: 114 unit: store_failed_payload file: lib/es/client.rb start line: 214 end line: 227 size: 12 LOC McCabe index: 2 number of parameters: 1 id: 115 unit: start file: spec/integration/timeouts/request_timeout_spec.rb start line: 27 end line: 38 size: 11 LOC McCabe index: 1 number of parameters: 0 id: 116 unit: capture_output file: spec/support/cli_helpers.rb start line: 13 end line: 24 size: 11 LOC McCabe index: 2 number of parameters: 0 id: 117 unit: capture_error file: spec/support/cli_helpers.rb start line: 25 end line: 36 size: 11 LOC McCabe index: 2 number of parameters: 0 id: 118 unit: write_license_to_file file: script/licenses/generate_notice.rb start line: 22 end line: 33 size: 11 LOC McCabe index: 3 number of parameters: 4 id: 119 unit: purge_crawls_allowed? file: lib/crawler/coordinator.rb start line: 138 end line: 151 size: 11 LOC McCabe index: 4 number of parameters: 0 id: 120 unit: add_default_root_certificates file: lib/crawler/http_client.rb start line: 256 end line: 272 size: 11 LOC McCabe index: 2 number of parameters: 1 id: 121 unit: initialize file: lib/crawler/data/rule.rb start line: 19 end line: 32 size: 11 LOC McCabe index: 3 number of parameters: 3 id: 122 unit: extract_links file: lib/crawler/data/crawl_result/sitemap.rb start line: 22 end line: 33 size: 11 LOC McCabe index: 3 number of parameters: 0 id: 123 unit: initialize file: lib/crawler/data/crawl_result/http_auth_disallowed_error.rb start line: 14 end line: 26 size: 11 LOC McCabe index: 3 number of parameters: 2 id: 124 unit: html_fields file: lib/crawler/document_mapper.rb start line: 67 end line: 78 size: 11 LOC McCabe index: 1 number of parameters: 1 id: 125 unit: validation_fail_from_crawl_error file: lib/crawler/url_validator/url_content_check_concern.rb start line: 66 end line: 79 size: 11 LOC McCabe index: 2 number of parameters: 0 id: 126 unit: initialize file: lib/crawler/url_validator.rb start line: 68 end line: 84 size: 11 LOC McCabe index: 4 number of parameters: 3 id: 127 unit: headers file: lib/crawler/http_utils/response.rb start line: 74 end line: 86 size: 11 LOC McCabe index: 4 number of parameters: 0 id: 128 unit: detect_encoding_from_content_charset file: lib/crawler/http_utils/response.rb start line: 123 end line: 136 size: 11 LOC McCabe index: 4 number of parameters: 0 id: 129 unit: configure_crawl_rules! file: lib/crawler/api/config.rb start line: 328 end line: 341 size: 11 LOC McCabe index: 4 number of parameters: 0 id: 130 unit: reset_ingestion_stats file: lib/crawler/output_sink/elasticsearch.rb start line: 264 end line: 276 size: 11 LOC McCabe index: 2 number of parameters: 1 id: 131 unit: coerce_to_absolute_urls file: spec/support/faux/faux_crawl.rb start line: 204 end line: 214 size: 10 LOC McCabe index: 2 number of parameters: 1 id: 132 unit: add_urls_to_backlog file: spec/lib/crawler/coordinator_spec.rb start line: 447 end line: 457 size: 10 LOC McCabe index: 1 number of parameters: 2 id: 133 unit: add_url_to_backlog file: spec/lib/crawler/coordinator_spec.rb start line: 522 end line: 532 size: 10 LOC McCabe index: 1 number of parameters: 1 id: 134 unit: expect_denied_with_reason file: spec/lib/crawler/coordinator_spec.rb start line: 651 end line: 661 size: 10 LOC McCabe index: 1 number of parameters: 2 id: 135 unit: fetch_valid_auto_discovered_sitemap_urls! file: lib/crawler/coordinator.rb start line: 250 end line: 261 size: 10 LOC McCabe index: 2 number of parameters: 0 id: 136 unit: enqueue_redirect_link file: lib/crawler/coordinator.rb start line: 434 end line: 444 size: 10 LOC McCabe index: 1 number of parameters: 2 id: 137 unit: initialize file: lib/crawler/http_client.rb start line: 67 end line: 80 size: 10 LOC McCabe index: 1 number of parameters: 1 id: 138 unit: new_connection_manager file: lib/crawler/http_client.rb start line: 199 end line: 209 size: 10 LOC McCabe index: 1 number of parameters: 0 id: 139 unit: ssl_hostname_verifier file: lib/crawler/http_client.rb start line: 275 end line: 285 size: 10 LOC McCabe index: 3 number of parameters: 0 id: 140 unit: format_logger file: lib/crawler/logging/handler/file.rb start line: 51 end line: 61 size: 10 LOC McCabe index: 2 number of parameters: 2 id: 141 unit: format_logger file: lib/crawler/logging/handler/stdout.rb start line: 49 end line: 59 size: 10 LOC McCabe index: 2 number of parameters: 2 id: 142 unit: extract_by_selector file: lib/crawler/data/crawl_result/html.rb start line: 210 end line: 220 size: 10 LOC McCabe index: 2 number of parameters: 2 id: 143 unit: initialize file: lib/crawler/data/crawl_result/base.rb start line: 21 end line: 33 size: 10 LOC McCabe index: 2 number of parameters: 5 id: 144 unit: initialize file: lib/crawler/data/crawl_result/redirect.rb start line: 18 end line: 31 size: 10 LOC McCabe index: 3 number of parameters: 4 id: 145 unit: rules file: lib/crawler/data/extraction/ruleset.rb start line: 26 end line: 36 size: 10 LOC McCabe index: 3 number of parameters: 0 id: 146 unit: url_filters file: lib/crawler/data/extraction/ruleset.rb start line: 37 end line: 47 size: 10 LOC McCabe index: 3 number of parameters: 0 id: 147 unit: call file: lib/crawler/cli/validate.rb start line: 18 end line: 30 size: 10 LOC McCabe index: 1 number of parameters: 3 id: 148 unit: print_validation_result file: lib/crawler/cli/validate.rb start line: 33 end line: 43 size: 10 LOC McCabe index: 3 number of parameters: 2 id: 149 unit: self.dedot_hash file: lib/crawler/cli/helpers.rb start line: 41 end line: 54 size: 10 LOC McCabe index: 3 number of parameters: 1 id: 150 unit: configure_domain_allowlist! file: lib/crawler/api/config.rb start line: 307 end line: 320 size: 10 LOC McCabe index: 4 number of parameters: 0 id: 151 unit: configure_sitemap_urls! file: lib/crawler/api/config.rb start line: 381 end line: 393 size: 10 LOC McCabe index: 4 number of parameters: 0 id: 152 unit: close file: lib/crawler/output_sink/elasticsearch.rb start line: 147 end line: 157 size: 10 LOC McCabe index: 1 number of parameters: 0 id: 153 unit: write file: lib/crawler/output_sink/console.rb start line: 13 end line: 25 size: 10 LOC McCabe index: 2 number of parameters: 1 id: 154 unit: log_error file: lib/crawler/event_generator.rb start line: 42 end line: 52 size: 10 LOC McCabe index: 1 number of parameters: 2 id: 155 unit: crawl_end file: lib/crawler/event_generator.rb start line: 84 end line: 94 size: 10 LOC McCabe index: 1 number of parameters: 3 id: 156 unit: crawl_status file: lib/crawler/event_generator.rb start line: 105 end line: 116 size: 10 LOC McCabe index: 1 number of parameters: 1 id: 157 unit: site_with_sitemap file: spec/integration/legacy_sitemaps_spec.rb start line: 11 end line: 21 size: 9 LOC McCabe index: 2 number of parameters: 1 id: 158 unit: configure_sites file: spec/support/faux/faux_crawl.rb start line: 81 end line: 92 size: 9 LOC McCabe index: 2 number of parameters: 1 id: 159 unit: check_discovered_url file: spec/lib/crawler/coordinator_spec.rb start line: 637 end line: 646 size: 9 LOC McCabe index: 1 number of parameters: 4 id: 160 unit: initialize file: lib/crawler/stats.rb start line: 16 end line: 30 size: 9 LOC McCabe index: 1 number of parameters: 0 id: 161 unit: run_crawl! file: lib/crawler/coordinator.rb start line: 72 end line: 86 size: 9 LOC McCabe index: 2 number of parameters: 0 id: 162 unit: enqueue_seed_urls file: lib/crawler/coordinator.rb start line: 204 end line: 213 size: 9 LOC McCabe index: 1 number of parameters: 0 id: 163 unit: initialize file: lib/crawler/robots_txt_parser.rb start line: 23 end line: 32 size: 9 LOC McCabe index: 1 number of parameters: 3 id: 164 unit: self.extract_from_crawl_result file: lib/crawler/content_engine/extractor.rb start line: 42 end line: 53 size: 9 LOC McCabe index: 3 number of parameters: 2 id: 165 unit: to_h file: lib/crawler/data/crawl_result/base.rb start line: 46 end line: 55 size: 9 LOC McCabe index: 1 number of parameters: 0 id: 166 unit: initialize file: lib/crawler/data/extraction/rule.rb start line: 28 end line: 37 size: 9 LOC McCabe index: 1 number of parameters: 1 id: 167 unit: validate_css file: lib/crawler/data/extraction/rule.rb start line: 84 end line: 93 size: 9 LOC McCabe index: 2 number of parameters: 1 id: 168 unit: validate_xpath file: lib/crawler/data/extraction/rule.rb start line: 94 end line: 103 size: 9 LOC McCabe index: 2 number of parameters: 1 id: 169 unit: crawl_rules_outcome file: lib/crawler/rule_engine/base.rb start line: 38 end line: 48 size: 9 LOC McCabe index: 4 number of parameters: 1 id: 170 unit: create_doc file: lib/crawler/document_mapper.rb start line: 18 end line: 31 size: 9 LOC McCabe index: 4 number of parameters: 1 id: 171 unit: create_html_doc file: lib/crawler/document_mapper.rb start line: 34 end line: 43 size: 9 LOC McCabe index: 1 number of parameters: 1 id: 172 unit: self.load_crawl_config file: lib/crawler/cli/helpers.rb start line: 25 end line: 39 size: 9 LOC McCabe index: 3 number of parameters: 2 id: 173 unit: generate_unexpected_type_crawl_result file: lib/crawler/http_executor.rb start line: 272 end line: 281 size: 9 LOC McCabe index: 2 number of parameters: 2 id: 174 unit: build_crawl_rules file: lib/crawler/api/config.rb start line: 342 end line: 351 size: 9 LOC McCabe index: 1 number of parameters: 2 id: 175 unit: record_overall_outcome file: lib/crawler/api/crawl.rb start line: 159 end line: 170 size: 9 LOC McCabe index: 3 number of parameters: 1 id: 176 unit: record_outcome file: lib/crawler/api/crawl.rb start line: 179 end line: 189 size: 9 LOC McCabe index: 1 number of parameters: 2 id: 177 unit: crawl_stage_end file: lib/crawler/event_generator.rb start line: 74 end line: 83 size: 9 LOC McCabe index: 1 number of parameters: 2 id: 178 unit: static_ecs_common_fields file: lib/crawler/event_generator.rb start line: 275 end line: 284 size: 9 LOC McCabe index: 2 number of parameters: 0 id: 179 unit: log file: lib/crawler/event_generator.rb start line: 341 end line: 356 size: 9 LOC McCabe index: 4 number of parameters: 1 id: 180 unit: run file: spec/support/faux/faux_crawl.rb start line: 135 end line: 149 size: 8 LOC McCabe index: 3 number of parameters: 0 id: 181 unit: with_example_site file: spec/lib/crawler/http_client_spec.rb start line: 39 end line: 47 size: 8 LOC McCabe index: 1 number of parameters: 0 id: 182 unit: count_url_discover file: lib/crawler/stats.rb start line: 91 end line: 99 size: 8 LOC McCabe index: 2 number of parameters: 1 id: 183 unit: run_primary_crawl! file: lib/crawler/coordinator.rb start line: 87 end line: 97 size: 8 LOC McCabe index: 1 number of parameters: 0 id: 184 unit: load_robots_txts file: lib/crawler/coordinator.rb start line: 162 end line: 171 size: 8 LOC McCabe index: 3 number of parameters: 0 id: 185 unit: execute_crawl_task file: lib/crawler/coordinator.rb start line: 354 end line: 365 size: 8 LOC McCabe index: 4 number of parameters: 1 id: 186 unit: default_request_config file: lib/crawler/http_client.rb start line: 306 end line: 314 size: 8 LOC McCabe index: 1 number of parameters: 0 id: 187 unit: credentials_provider file: lib/crawler/http_client.rb start line: 337 end line: 346 size: 8 LOC McCabe index: 3 number of parameters: 0 id: 188 unit: initialize file: lib/crawler/logging/handler/file.rb start line: 16 end line: 28 size: 8 LOC McCabe index: 3 number of parameters: 3 id: 189 unit: self.extract file: lib/crawler/content_engine/extractor.rb start line: 13 end line: 24 size: 8 LOC McCabe index: 2 number of parameters: 2 id: 190 unit: self.match_url_filters? file: lib/crawler/content_engine/extractor.rb start line: 60 end line: 71 size: 8 LOC McCabe index: 2 number of parameters: 2 id: 191 unit: self.transform! file: lib/crawler/content_engine/transformer.rb start line: 19 end line: 29 size: 8 LOC McCabe index: 2 number of parameters: 1 id: 192 unit: extract_by_regexp file: lib/crawler/data/url.rb start line: 75 end line: 88 size: 8 LOC McCabe index: 4 number of parameters: 1 id: 193 unit: coerce_to_links file: lib/crawler/data/crawl_result/sitemap.rb start line: 35 end line: 44 size: 8 LOC McCabe index: 1 number of parameters: 1 id: 194 unit: initialize file: lib/crawler/data/crawl_result/success.rb start line: 19 end line: 29 size: 8 LOC McCabe index: 2 number of parameters: 3 id: 195 unit: validate_actions file: lib/crawler/data/extraction/rule.rb start line: 48 end line: 58 size: 8 LOC McCabe index: 5 number of parameters: 0 id: 196 unit: to_h file: lib/crawler/data/crawl_task.rb start line: 61 end line: 69 size: 8 LOC McCabe index: 1 number of parameters: 0 id: 197 unit: self.load file: lib/crawler/data/crawl_task.rb start line: 71 end line: 79 size: 8 LOC McCabe index: 1 number of parameters: 1 id: 198 unit: register_crawl_result file: lib/crawler/robots_txt_service.rb start line: 29 end line: 37 size: 8 LOC McCabe index: 2 number of parameters: 2 id: 199 unit: create_binary_file_doc file: lib/crawler/document_mapper.rb start line: 44 end line: 52 size: 8 LOC McCabe index: 1 number of parameters: 1 id: 200 unit: binary_file_fields file: lib/crawler/document_mapper.rb start line: 79 end line: 87 size: 8 LOC McCabe index: 1 number of parameters: 1 id: 201 unit: self.load_yaml file: lib/crawler/cli/helpers.rb start line: 16 end line: 24 size: 8 LOC McCabe index: 3 number of parameters: 1 id: 202 unit: unsupported_content_type file: lib/crawler/http_executor.rb start line: 168 end line: 176 size: 8 LOC McCabe index: 2 number of parameters: 2 id: 203 unit: validation_warn_from_crawl_redirect file: lib/crawler/url_validator/url_content_check_concern.rb start line: 56 end line: 64 size: 8 LOC McCabe index: 1 number of parameters: 0 id: 204 unit: validation_ok file: lib/crawler/url_validator.rb start line: 128 end line: 136 size: 8 LOC McCabe index: 1 number of parameters: 3 id: 205 unit: validation_warn file: lib/crawler/url_validator.rb start line: 137 end line: 145 size: 8 LOC McCabe index: 1 number of parameters: 3 id: 206 unit: validation_fail file: lib/crawler/url_validator.rb start line: 146 end line: 154 size: 8 LOC McCabe index: 1 number of parameters: 3 id: 207 unit: initialize file: lib/crawler/http_utils/response.rb start line: 23 end line: 34 size: 8 LOC McCabe index: 2 number of parameters: 4 id: 208 unit: suggestion_message file: lib/crawler/http_utils/exceptions.rb start line: 194 end line: 202 size: 8 LOC McCabe index: 1 number of parameters: 0 id: 209 unit: configure_logging! file: lib/crawler/api/config.rb start line: 411 end line: 423 size: 8 LOC McCabe index: 4 number of parameters: 3 id: 210 unit: interruptible_sleep file: lib/crawler/api/crawl.rb start line: 69 end line: 78 size: 8 LOC McCabe index: 3 number of parameters: 1 id: 211 unit: verify_output_index file: lib/crawler/output_sink/elasticsearch.rb start line: 65 end line: 73 size: 8 LOC McCabe index: 2 number of parameters: 0 id: 212 unit: pipeline_enabled? file: lib/crawler/output_sink/elasticsearch.rb start line: 213 end line: 221 size: 8 LOC McCabe index: 3 number of parameters: 0 id: 213 unit: crawl_seed file: lib/crawler/event_generator.rb start line: 95 end line: 103 size: 8 LOC McCabe index: 1 number of parameters: 2 id: 214 unit: bulk file: lib/es/client.rb start line: 61 end line: 69 size: 8 LOC McCabe index: 2 number of parameters: 1 id: 215 unit: process_crawl_result file: spec/lib/crawler/coordinator_spec.rb start line: 180 end line: 188 size: 7 LOC McCabe index: 1 number of parameters: 0 id: 216 unit: with_proxy file: spec/lib/crawler/http_client_spec.rb start line: 150 end line: 157 size: 7 LOC McCabe index: 1 number of parameters: 0 id: 217 unit: with_mock_server file: spec/lib/crawler/http_client_spec.rb start line: 246 end line: 253 size: 7 LOC McCabe index: 1 number of parameters: 0 id: 218 unit: write_header_to_file file: script/licenses/generate_notice.rb start line: 14 end line: 21 size: 7 LOC McCabe index: 1 number of parameters: 1 id: 219 unit: count_url_fetch file: lib/crawler/stats.rb start line: 101 end line: 109 size: 7 LOC McCabe index: 1 number of parameters: 1 id: 220 unit: execute_task file: lib/crawler/coordinator.rb start line: 368 end line: 376 size: 7 LOC McCabe index: 2 number of parameters: 2 id: 221 unit: extract_and_enqueue_links file: lib/crawler/coordinator.rb start line: 423 end line: 432 size: 7 LOC McCabe index: 6 number of parameters: 2 id: 222 unit: dns_resolver file: lib/crawler/http_client.rb start line: 288 end line: 295 size: 7 LOC McCabe index: 1 number of parameters: 0 id: 223 unit: proxy_credentials file: lib/crawler/http_client.rb start line: 349 end line: 357 size: 7 LOC McCabe index: 3 number of parameters: 0 id: 224 unit: initialize file: lib/crawler/logging/handler/stdout.rb start line: 16 end line: 26 size: 7 LOC McCabe index: 1 number of parameters: 1 id: 225 unit: to_url file: lib/crawler/data/link.rb start line: 61 end line: 69 size: 7 LOC McCabe index: 3 number of parameters: 0 id: 226 unit: valid? file: lib/crawler/data/link.rb start line: 73 end line: 80 size: 7 LOC McCabe index: 2 number of parameters: 0 id: 227 unit: sitemap_parser file: lib/crawler/data/crawl_result/sitemap.rb start line: 47 end line: 54 size: 7 LOC McCabe index: 2 number of parameters: 0 id: 228 unit: document_body file: lib/crawler/data/crawl_result/html.rb start line: 177 end line: 185 size: 7 LOC McCabe index: 2 number of parameters: 1 id: 229 unit: initialize file: lib/crawler/data/crawl_result/redirect_error.rb start line: 14 end line: 22 size: 7 LOC McCabe index: 1 number of parameters: 1 id: 230 unit: crawl_rule_denied file: lib/crawler/data/rule_engine_outcome.rb start line: 50 end line: 57 size: 7 LOC McCabe index: 1 number of parameters: 1 id: 231 unit: validate_rule file: lib/crawler/data/extraction/rule.rb start line: 40 end line: 47 size: 7 LOC McCabe index: 1 number of parameters: 0 id: 232 unit: validate_field_name file: lib/crawler/data/extraction/rule.rb start line: 59 end line: 69 size: 7 LOC McCabe index: 4 number of parameters: 0 id: 233 unit: initialize file: lib/crawler/data/extraction/ruleset.rb start line: 16 end line: 25 size: 7 LOC McCabe index: 1 number of parameters: 2 id: 234 unit: validate_ruleset file: lib/crawler/data/extraction/ruleset.rb start line: 57 end line: 66 size: 7 LOC McCabe index: 5 number of parameters: 0 id: 235 unit: sitemaps file: lib/crawler/robots_txt_service.rb start line: 60 end line: 67 size: 7 LOC McCabe index: 1 number of parameters: 0 id: 236 unit: call file: lib/crawler/cli/schedule.rb start line: 21 end line: 29 size: 7 LOC McCabe index: 3 number of parameters: 3 id: 237 unit: mock_404_result file: lib/crawler/mock_executor.rb start line: 30 end line: 37 size: 7 LOC McCabe index: 1 number of parameters: 1 id: 238 unit: http_client_status file: lib/crawler/http_executor.rb start line: 29 end line: 36 size: 7 LOC McCabe index: 1 number of parameters: 0 id: 239 unit: timeout_error file: lib/crawler/http_executor.rb start line: 283 end line: 290 size: 7 LOC McCabe index: 1 number of parameters: 3 id: 240 unit: validate_domain_uniqueness file: lib/crawler/url_validator/domain_uniqueness_check_concern.rb start line: 12 end line: 19 size: 7 LOC McCabe index: 2 number of parameters: 0 id: 241 unit: validate_domain_access file: lib/crawler/url_validator/domain_access_check_concern.rb start line: 12 end line: 19 size: 7 LOC McCabe index: 2 number of parameters: 0 id: 242 unit: validate file: lib/crawler/url_validator.rb start line: 100 end line: 107 size: 7 LOC McCabe index: 2 number of parameters: 0 id: 243 unit: initialize file: lib/crawler/http_utils/config.rb start line: 47 end line: 55 size: 7 LOC McCabe index: 3 number of parameters: 1 id: 244 unit: suggestion_message file: lib/crawler/http_utils/exceptions.rb start line: 41 end line: 48 size: 7 LOC McCabe index: 1 number of parameters: 0 id: 245 unit: to_s file: lib/crawler/api/config.rb start line: 245 end line: 252 size: 7 LOC McCabe index: 1 number of parameters: 0 id: 246 unit: parse_certificate_string file: lib/crawler/api/config.rb start line: 290 end line: 297 size: 7 LOC McCabe index: 4 number of parameters: 1 id: 247 unit: load_certificate_from_file file: lib/crawler/api/config.rb start line: 299 end line: 306 size: 7 LOC McCabe index: 4 number of parameters: 1 id: 248 unit: start_shutdown! file: lib/crawler/api/crawl.rb start line: 59 end line: 66 size: 7 LOC McCabe index: 1 number of parameters: 2 id: 249 unit: combined_outcome file: lib/crawler/api/crawl.rb start line: 171 end line: 178 size: 7 LOC McCabe index: 3 number of parameters: 1 id: 250 unit: print_extracted_links file: lib/crawler/api/crawl.rb start line: 244 end line: 252 size: 7 LOC McCabe index: 2 number of parameters: 1 id: 251 unit: operation_queue file: lib/crawler/output_sink/elasticsearch.rb start line: 188 end line: 195 size: 7 LOC McCabe index: 2 number of parameters: 0 id: 252 unit: assign_default_pipeline file: lib/crawler/output_sink/elasticsearch.rb start line: 228 end line: 236 size: 7 LOC McCabe index: 3 number of parameters: 2 id: 253 unit: ecs_common_fields file: lib/crawler/event_generator.rb start line: 285 end line: 292 size: 7 LOC McCabe index: 1 number of parameters: 0 id: 254 unit: log_final_failure file: lib/es/client.rb start line: 249 end line: 256 size: 7 LOC McCabe index: 2 number of parameters: 3 id: 255 unit: log_crawl_end_event file: lib/crawler/coordinator.rb start line: 739 end line: 745 size: 6 LOC McCabe index: 1 number of parameters: 0 id: 256 unit: normalized_url file: lib/crawler/data/url.rb start line: 30 end line: 36 size: 6 LOC McCabe index: 2 number of parameters: 0 id: 257 unit: self.queue_class_for_type file: lib/crawler/data/url_queue.rb start line: 22 end line: 28 size: 6 LOC McCabe index: 2 number of parameters: 1 id: 258 unit: sitemap file: lib/crawler/data/crawl_result/sitemap.rb start line: 56 end line: 62 size: 6 LOC McCabe index: 2 number of parameters: 0 id: 259 unit: valid_field_name? file: lib/crawler/data/crawl_result/html.rb start line: 155 end line: 168 size: 6 LOC McCabe index: 5 number of parameters: 1 id: 260 unit: to_h file: lib/crawler/data/crawl_result/redirect.rb start line: 36 end line: 42 size: 6 LOC McCabe index: 1 number of parameters: 0 id: 261 unit: noindex_meta_denied file: lib/crawler/data/rule_engine_outcome.rb start line: 32 end line: 38 size: 6 LOC McCabe index: 1 number of parameters: 0 id: 262 unit: domain_filter_denied file: lib/crawler/data/rule_engine_outcome.rb start line: 39 end line: 45 size: 6 LOC McCabe index: 1 number of parameters: 1 id: 263 unit: crawl_rule_allowed file: lib/crawler/data/rule_engine_outcome.rb start line: 58 end line: 64 size: 6 LOC McCabe index: 1 number of parameters: 1 id: 264 unit: crawl_rule_timeout file: lib/crawler/data/rule_engine_outcome.rb start line: 65 end line: 71 size: 6 LOC McCabe index: 3 number of parameters: 1 id: 265 unit: url_filtering_rules file: lib/crawler/data/extraction/ruleset.rb start line: 48 end line: 54 size: 6 LOC McCabe index: 2 number of parameters: 0 id: 266 unit: setup_memory_queue file: lib/crawler/data/url_queue/memory_only.rb start line: 26 end line: 34 size: 6 LOC McCabe index: 4 number of parameters: 0 id: 267 unit: initialize file: lib/crawler/data/domain.rb start line: 13 end line: 19 size: 6 LOC McCabe index: 2 number of parameters: 1 id: 268 unit: standard_port_for_scheme file: lib/crawler/data/domain.rb start line: 28 end line: 34 size: 6 LOC McCabe index: 3 number of parameters: 1 id: 269 unit: initialize file: lib/crawler/data/crawl_task.rb start line: 20 end line: 26 size: 6 LOC McCabe index: 1 number of parameters: 4 id: 270 unit: headers file: lib/crawler/data/crawl_task.rb start line: 43 end line: 50 size: 6 LOC McCabe index: 2 number of parameters: 0 id: 271 unit: meta_tags_and_data_attributes file: lib/crawler/document_mapper.rb start line: 53 end line: 59 size: 6 LOC McCabe index: 1 number of parameters: 1 id: 272 unit: core_fields file: lib/crawler/document_mapper.rb start line: 60 end line: 66 size: 6 LOC McCabe index: 1 number of parameters: 1 id: 273 unit: self.find_or_create_target_hash file: lib/crawler/cli/helpers.rb start line: 56 end line: 62 size: 6 LOC McCabe index: 3 number of parameters: 2 id: 274 unit: self.sink_class_for_type file: lib/crawler/output_sink.rb start line: 15 end line: 21 size: 6 LOC McCabe index: 2 number of parameters: 1 id: 275 unit: initialize file: lib/crawler/url_validator/result.rb start line: 12 end line: 18 size: 6 LOC McCabe index: 1 number of parameters: 4 id: 276 unit: check_content_encoding file: lib/crawler/http_utils/response.rb start line: 200 end line: 207 size: 6 LOC McCabe index: 2 number of parameters: 0 id: 277 unit: initialize file: lib/crawler/http_utils/filtering_dns_resolver.rb start line: 20 end line: 26 size: 6 LOC McCabe index: 1 number of parameters: 3 id: 278 unit: validate_domain! file: lib/crawler/api/config.rb start line: 321 end line: 327 size: 6 LOC McCabe index: 4 number of parameters: 1 id: 279 unit: initialize file: lib/crawler/output_sink/file.rb start line: 15 end line: 23 size: 6 LOC McCabe index: 3 number of parameters: 1 id: 280 unit: write file: lib/crawler/output_sink/file.rb start line: 24 end line: 31 size: 6 LOC McCabe index: 1 number of parameters: 1 id: 281 unit: log_crawl_status file: lib/crawler/event_generator.rb start line: 33 end line: 40 size: 6 LOC McCabe index: 3 number of parameters: 2 id: 282 unit: static_crawl_fields file: lib/crawler/event_generator.rb start line: 309 end line: 315 size: 6 LOC McCabe index: 2 number of parameters: 0 id: 283 unit: will_fit? file: lib/es/bulk_queue.rb start line: 65 end line: 73 size: 6 LOC McCabe index: 2 number of parameters: 2 id: 284 unit: current_stats file: lib/es/bulk_queue.rb start line: 74 end line: 80 size: 6 LOC McCabe index: 1 number of parameters: 0 id: 285 unit: reset file: lib/es/bulk_queue.rb start line: 95 end line: 102 size: 6 LOC McCabe index: 1 number of parameters: 0 id: 286 unit: equal_for_specified_keys? file: spec/support/mock_response.rb start line: 11 end line: 16 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 287 unit: fixture_xml file: spec/support/fixtures.rb start line: 18 end line: 23 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 288 unit: fixture_xml_gz file: spec/support/fixtures.rb start line: 24 end line: 29 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 289 unit: stub_http_resolver! file: spec/lib/crawler/http_client_spec.rb start line: 32 end line: 37 size: 5 LOC McCabe index: 1 number of parameters: 0 id: 290 unit: expect_result file: spec/lib/crawler/http_client_spec.rb start line: 54 end line: 59 size: 5 LOC McCabe index: 1 number of parameters: 2 id: 291 unit: crawl_duration_msec file: lib/crawler/stats.rb start line: 38 end line: 44 size: 5 LOC McCabe index: 3 number of parameters: 0 id: 292 unit: crawl_task_progress file: lib/crawler/coordinator.rb start line: 154 end line: 159 size: 5 LOC McCabe index: 1 number of parameters: 2 id: 293 unit: enqueue_purge_urls file: lib/crawler/coordinator.rb start line: 244 end line: 249 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 294 unit: https_socket_factory file: lib/crawler/http_client.rb start line: 211 end line: 219 size: 5 LOC McCabe index: 1 number of parameters: 0 id: 295 unit: add_custom_ca_certificates file: lib/crawler/http_client.rb start line: 247 end line: 252 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 296 unit: default_socket_config file: lib/crawler/http_client.rb start line: 298 end line: 303 size: 5 LOC McCabe index: 1 number of parameters: 0 id: 297 unit: self.robots_txt_to_byte_array file: lib/crawler/robots_txt_parser.rb start line: 12 end line: 22 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 298 unit: robots_rules_parser file: lib/crawler/robots_txt_parser.rb start line: 61 end line: 67 size: 5 LOC McCabe index: 1 number of parameters: 0 id: 299 unit: initialize file: lib/crawler/robots_txt_parser.rb start line: 69 end line: 74 size: 5 LOC McCabe index: 1 number of parameters: 2 id: 300 unit: route_logs_to_handlers file: lib/crawler/logging/logger.rb start line: 21 end line: 26 size: 5 LOC McCabe index: 1 number of parameters: 2 id: 301 unit: add_tags_to_log_handlers file: lib/crawler/logging/logger.rb start line: 59 end line: 64 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 302 unit: initialize file: lib/crawler/logging/handler/base.rb start line: 13 end line: 18 size: 5 LOC McCabe index: 1 number of parameters: 3 id: 303 unit: url_match? file: lib/crawler/data/rule.rb start line: 33 end line: 38 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 304 unit: domain_name file: lib/crawler/data/url.rb start line: 43 end line: 48 size: 5 LOC McCabe index: 2 number of parameters: 0 id: 305 unit: xml_content file: lib/crawler/data/crawl_result/sitemap.rb start line: 65 end line: 70 size: 5 LOC McCabe index: 2 number of parameters: 0 id: 306 unit: to_h file: lib/crawler/data/crawl_result/html.rb start line: 30 end line: 35 size: 5 LOC McCabe index: 1 number of parameters: 0 id: 307 unit: links file: lib/crawler/data/crawl_result/html.rb start line: 81 end line: 89 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 308 unit: document_title file: lib/crawler/data/crawl_result/html.rb start line: 170 end line: 175 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 309 unit: initialize file: lib/crawler/data/crawl_result/content_extractable_file.rb start line: 21 end line: 27 size: 5 LOC McCabe index: 1 number of parameters: 4 id: 310 unit: initialize file: lib/crawler/data/crawl_result/error.rb start line: 21 end line: 26 size: 5 LOC McCabe index: 1 number of parameters: 4 id: 311 unit: initialize file: lib/crawler/data/rule_engine_outcome.rb start line: 87 end line: 92 size: 5 LOC McCabe index: 1 number of parameters: 3 id: 312 unit: validate_join_as file: lib/crawler/data/extraction/rule.rb start line: 70 end line: 76 size: 5 LOC McCabe index: 3 number of parameters: 0 id: 313 unit: validate_source file: lib/crawler/data/extraction/rule.rb start line: 77 end line: 83 size: 5 LOC McCabe index: 2 number of parameters: 0 id: 314 unit: initialize file: lib/crawler/data/extraction/url_filter.rb start line: 17 end line: 22 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 315 unit: validate_regex file: lib/crawler/data/extraction/url_filter.rb start line: 43 end line: 48 size: 5 LOC McCabe index: 2 number of parameters: 0 id: 316 unit: maybe_threshold_alert file: lib/crawler/data/url_queue/base.rb start line: 52 end line: 58 size: 5 LOC McCabe index: 2 number of parameters: 1 id: 317 unit: call file: lib/crawler/cli/urltest.rb start line: 21 end line: 27 size: 5 LOC McCabe index: 1 number of parameters: 4 id: 318 unit: call file: lib/crawler/cli/crawl.rb start line: 20 end line: 26 size: 5 LOC McCabe index: 1 number of parameters: 3 id: 319 unit: to_h file: lib/crawler/url_validator/result.rb start line: 23 end line: 28 size: 5 LOC McCabe index: 2 number of parameters: 0 id: 320 unit: normalized_url file: lib/crawler/url_validator.rb start line: 112 end line: 117 size: 5 LOC McCabe index: 2 number of parameters: 0 id: 321 unit: perform_check file: lib/crawler/url_validator.rb start line: 155 end line: 161 size: 5 LOC McCabe index: 2 number of parameters: 1 id: 322 unit: validate_required_options file: lib/crawler/http_utils/config.rb start line: 130 end line: 135 size: 5 LOC McCabe index: 2 number of parameters: 2 id: 323 unit: create_response_buffer file: lib/crawler/http_utils/response.rb start line: 139 end line: 144 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 324 unit: allowed_address? file: lib/crawler/http_utils/filtering_dns_resolver.rb start line: 60 end line: 66 size: 5 LOC McCabe index: 5 number of parameters: 1 id: 325 unit: local_address? file: lib/crawler/http_utils/filtering_dns_resolver.rb start line: 68 end line: 73 size: 5 LOC McCabe index: 3 number of parameters: 1 id: 326 unit: initialize file: lib/crawler/http_utils/exceptions.rb start line: 59 end line: 64 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 327 unit: assign_config_params file: lib/crawler/api/config.rb start line: 258 end line: 263 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 328 unit: initialize file: lib/crawler/output_sink/mock.rb start line: 15 end line: 21 size: 5 LOC McCabe index: 2 number of parameters: 1 id: 329 unit: parametrized_doc file: lib/crawler/output_sink/elasticsearch.rb start line: 237 end line: 242 size: 5 LOC McCabe index: 2 number of parameters: 1 id: 330 unit: outcome_from_status_code file: lib/crawler/event_generator.rb start line: 186 end line: 192 size: 5 LOC McCabe index: 5 number of parameters: 1 id: 331 unit: pop_all file: lib/es/bulk_queue.rb start line: 34 end line: 41 size: 5 LOC McCabe index: 1 number of parameters: 0 id: 332 unit: initialize file: lib/es/client.rb start line: 33 end line: 39 size: 5 LOC McCabe index: 1 number of parameters: 5 id: 333 unit: delete_by_query file: lib/es/client.rb start line: 88 end line: 93 size: 5 LOC McCabe index: 1 number of parameters: 3 id: 334 unit: configure_compression file: lib/es/client.rb start line: 177 end line: 182 size: 5 LOC McCabe index: 1 number of parameters: 1 id: 335 unit: response_body file: spec/integration/timeouts/socket_timeout_spec.rb start line: 18 end line: 23 size: 4 LOC McCabe index: 1 number of parameters: 0 id: 336 unit: self.crawl_site file: spec/support/faux/faux_crawl.rb start line: 31 end line: 36 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 337 unit: document_body file: spec/lib/crawler/content_engine/transformer_spec.rb start line: 13 end line: 17 size: 4 LOC McCabe index: 1 number of parameters: 0 id: 338 unit: normalize_license file: script/licenses/lib/third_party.rb start line: 33 end line: 38 size: 4 LOC McCabe index: 3 number of parameters: 1 id: 339 unit: initialize file: lib/crawler/http_header_service.rb start line: 62 end line: 67 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 340 unit: set_outcome file: lib/crawler/coordinator.rb start line: 263 end line: 267 size: 4 LOC McCabe index: 1 number of parameters: 2 id: 341 unit: crawl_delay file: lib/crawler/robots_txt_parser.rb start line: 33 end line: 37 size: 4 LOC McCabe index: 1 number of parameters: 0 id: 342 unit: allowed? file: lib/crawler/robots_txt_parser.rb start line: 42 end line: 46 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 343 unit: self.cast_result file: lib/crawler/content_engine/extractor.rb start line: 54 end line: 59 size: 4 LOC McCabe index: 2 number of parameters: 2 id: 344 unit: url_hash file: lib/crawler/data/seen_urls.rb start line: 41 end line: 46 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 345 unit: error file: lib/crawler/data/link.rb start line: 83 end line: 88 size: 4 LOC McCabe index: 2 number of parameters: 0 id: 346 unit: self.create file: lib/crawler/data/url_queue.rb start line: 17 end line: 21 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 347 unit: canonical_link file: lib/crawler/data/crawl_result/html.rb start line: 96 end line: 100 size: 4 LOC McCabe index: 2 number of parameters: 0 id: 348 unit: meta_keywords file: lib/crawler/data/crawl_result/html.rb start line: 112 end line: 116 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 349 unit: meta_description file: lib/crawler/data/crawl_result/html.rb start line: 118 end line: 122 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 350 unit: full_html file: lib/crawler/data/crawl_result/html.rb start line: 221 end line: 226 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 351 unit: initialize file: lib/crawler/data/rule_engine_outcome.rb start line: 103 end line: 107 size: 4 LOC McCabe index: 1 number of parameters: 2 id: 352 unit: initialize file: lib/crawler/data/url_queue/base.rb start line: 24 end line: 28 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 353 unit: time_since_last_threshold_alert file: lib/crawler/data/url_queue/base.rb start line: 44 end line: 49 size: 4 LOC McCabe index: 2 number of parameters: 0 id: 354 unit: push file: lib/crawler/data/url_queue/base.rb start line: 61 end line: 65 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 355 unit: initialize file: lib/crawler/data/url_queue/memory_only.rb start line: 20 end line: 24 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 356 unit: push file: lib/crawler/data/url_queue/memory_only.rb start line: 66 end line: 70 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 357 unit: initialize file: lib/crawler/rule_engine/base.rb start line: 15 end line: 20 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 358 unit: initialize file: lib/crawler/robots_txt_service.rb start line: 20 end line: 24 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 359 unit: extraction_rule_fields file: lib/crawler/document_mapper.rb start line: 103 end line: 107 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 360 unit: self.die file: lib/crawler/cli/helpers.rb start line: 11 end line: 15 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 361 unit: run file: lib/crawler/mock_executor.rb start line: 25 end line: 29 size: 4 LOC McCabe index: 1 number of parameters: 2 id: 362 unit: initialize file: lib/crawler/http_executor.rb start line: 23 end line: 27 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 363 unit: self.create file: lib/crawler/output_sink.rb start line: 10 end line: 14 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 364 unit: << file: lib/crawler/mock_event_logger.rb start line: 19 end line: 24 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 365 unit: valid? file: lib/crawler/url_validator.rb start line: 94 end line: 98 size: 4 LOC McCabe index: 2 number of parameters: 0 id: 366 unit: [] file: lib/crawler/http_utils/response.rb start line: 69 end line: 73 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 367 unit: response_content_encodings file: lib/crawler/http_utils/response.rb start line: 187 end line: 191 size: 4 LOC McCabe index: 1 number of parameters: 0 id: 368 unit: resolve file: lib/crawler/http_utils/filtering_dns_resolver.rb start line: 27 end line: 31 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 369 unit: self.for_proxy_host file: lib/crawler/http_utils/exceptions.rb start line: 77 end line: 81 size: 4 LOC McCabe index: 1 number of parameters: 2 id: 370 unit: validate_param_names! file: lib/crawler/api/config.rb start line: 253 end line: 257 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 371 unit: write file: lib/crawler/output_sink/mock.rb start line: 22 end line: 27 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 372 unit: attempt_index_creation_or_exit file: lib/crawler/output_sink/elasticsearch.rb start line: 74 end line: 79 size: 4 LOC McCabe index: 2 number of parameters: 0 id: 373 unit: pipeline file: lib/crawler/output_sink/elasticsearch.rb start line: 208 end line: 212 size: 4 LOC McCabe index: 3 number of parameters: 0 id: 374 unit: increment_ingestion_stats file: lib/crawler/output_sink/elasticsearch.rb start line: 259 end line: 263 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 375 unit: initialize file: lib/crawler/output_sink/base.rb start line: 17 end line: 21 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 376 unit: initialize file: lib/crawler/event_generator.rb start line: 19 end line: 25 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 377 unit: url_discover_denied file: lib/crawler/event_generator.rb start line: 207 end line: 212 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 378 unit: output_event file: lib/crawler/event_generator.rb start line: 357 end line: 362 size: 4 LOC McCabe index: 1 number of parameters: 1 id: 379 unit: duration_to_nanoseconds file: lib/crawler/event_generator.rb start line: 365 end line: 369 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 380 unit: bytesize file: lib/es/bulk_queue.rb start line: 81 end line: 86 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 381 unit: serialize file: lib/es/bulk_queue.rb start line: 89 end line: 94 size: 4 LOC McCabe index: 2 number of parameters: 1 id: 382 unit: initialize file: lib/es/client.rb start line: 25 end line: 29 size: 4 LOC McCabe index: 1 number of parameters: 2 id: 383 unit: root_url file: spec/integration/timeouts/request_timeout_spec.rb start line: 23 end line: 26 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 384 unit: response_body file: spec/integration/sitemap_xxe_spec.rb start line: 47 end line: 50 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 385 unit: response_body file: spec/integration/sitemap_xxe_spec.rb start line: 72 end line: 75 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 386 unit: response_body file: spec/integration/response_limits_spec.rb start line: 24 end line: 27 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 387 unit: response_body file: spec/integration/response_limits_spec.rb start line: 31 end line: 34 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 388 unit: response_status file: spec/integration/headers_spec.rb start line: 12 end line: 15 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 389 unit: response_status file: spec/integration/headers_spec.rb start line: 22 end line: 25 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 390 unit: response_status file: spec/integration/headers_spec.rb start line: 32 end line: 35 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 391 unit: mock_response file: spec/support/mock_response.rb start line: 18 end line: 21 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 392 unit: initialize file: spec/support/faux/results_collection.rb start line: 16 end line: 19 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 393 unit: dup file: spec/support/faux/results_collection.rb start line: 23 end line: 26 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 394 unit: method_missing file: spec/support/faux/results_collection.rb start line: 27 end line: 30 size: 3 LOC McCabe index: 1 number of parameters: 3 id: 395 unit: respond_to_missing? file: spec/support/faux/results_collection.rb start line: 31 end line: 34 size: 3 LOC McCabe index: 2 number of parameters: 2 id: 396 unit: self.faux_url file: spec/support/faux/faux_crawl.rb start line: 13 end line: 16 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 397 unit: self.faux_ip file: spec/support/faux/faux_crawl.rb start line: 17 end line: 20 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 398 unit: self.faux_port file: spec/support/faux/faux_crawl.rb start line: 21 end line: 24 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 399 unit: self.run file: spec/support/faux/faux_crawl.rb start line: 27 end line: 30 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 400 unit: enterprise_search? file: spec/support/faux/faux_crawl.rb start line: 76 end line: 79 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 401 unit: stop_sites file: spec/support/faux/faux_crawl.rb start line: 130 end line: 133 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 402 unit: log file: spec/support/faux/faux_crawl.rb start line: 215 end line: 218 size: 3 LOC McCabe index: 1 number of parameters: 2 id: 403 unit: fixture_file file: spec/support/fixtures.rb start line: 10 end line: 13 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 404 unit: read_fixture file: spec/support/fixtures.rb start line: 14 end line: 17 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 405 unit: validate_test_check file: spec/lib/crawler/url_validator_spec.rb start line: 136 end line: 139 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 406 unit: crawl_finished? file: spec/lib/crawler/coordinator_spec.rb start line: 577 end line: 580 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 407 unit: url file: spec/lib/crawler/coordinator_spec.rb start line: 647 end line: 650 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 408 unit: extract_links file: spec/lib/crawler/coordinator_spec.rb start line: 757 end line: 760 size: 3 LOC McCabe index: 1 number of parameters: 2 id: 409 unit: get file: spec/lib/crawler/http_client_spec.rb start line: 50 end line: 53 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 410 unit: expect_success file: spec/lib/crawler/http_client_spec.rb start line: 60 end line: 63 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 411 unit: url file: spec/lib/crawler/data/url_spec.rb start line: 9 end line: 12 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 412 unit: domain file: spec/lib/crawler/data/domain_spec.rb start line: 9 end line: 12 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 413 unit: check_content_encoding file: spec/lib/crawler/http_utils/response_spec.rb start line: 24 end line: 27 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 414 unit: get file: spec/lib/crawler/http_utils/bad_ssl_spec.rb start line: 24 end line: 27 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 415 unit: expect_x509_certificates file: spec/lib/crawler/api/config_spec.rb start line: 168 end line: 171 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 416 unit: new_sink file: spec/lib/crawler/output_sink/file_spec.rb start line: 12 end line: 15 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 417 unit: execute_retry file: spec/lib/es/client_spec.rb start line: 314 end line: 317 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 418 unit: colorize file: script/support/string_colors.rb start line: 4 end line: 7 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 419 unit: red file: script/support/string_colors.rb start line: 8 end line: 11 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 420 unit: green file: script/support/string_colors.rb start line: 12 end line: 15 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 421 unit: yellow file: script/support/string_colors.rb start line: 16 end line: 19 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 422 unit: self.version file: lib/crawler.rb start line: 10 end line: 13 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 423 unit: self.service_id file: lib/crawler.rb start line: 15 end line: 18 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 424 unit: hash_of_fixnums file: lib/crawler/stats.rb start line: 32 end line: 35 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 425 unit: fetched_pages_count file: lib/crawler/stats.rb start line: 46 end line: 49 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 426 unit: urls_allowed_count file: lib/crawler/stats.rb start line: 51 end line: 54 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 427 unit: urls_denied_counts file: lib/crawler/stats.rb start line: 56 end line: 59 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 428 unit: time_spent_crawling_msec file: lib/crawler/stats.rb start line: 61 end line: 64 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 429 unit: status_code_counts file: lib/crawler/stats.rb start line: 66 end line: 69 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 430 unit: average_response_time_msec file: lib/crawler/stats.rb start line: 71 end line: 74 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 431 unit: crawl_duration file: lib/crawler/coordinator.rb start line: 62 end line: 65 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 432 unit: active_threads file: lib/crawler/coordinator.rb start line: 67 end line: 70 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 433 unit: executors_available? file: lib/crawler/coordinator.rb start line: 270 end line: 273 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 434 unit: create file: lib/crawler/http_client.rb start line: 24 end line: 27 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 435 unit: connection_pool_stats file: lib/crawler/http_client.rb start line: 157 end line: 160 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 436 unit: content_decoders file: lib/crawler/http_client.rb start line: 194 end line: 197 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 437 unit: finalize file: lib/crawler/http_client.rb start line: 383 end line: 386 size: 3 LOC McCabe index: 1 number of parameters: 2 id: 438 unit: self.url_pattern file: lib/crawler/utils.rb start line: 10 end line: 13 size: 3 LOC McCabe index: 1 number of parameters: 3 id: 439 unit: self.pattern_with_wildcard file: lib/crawler/utils.rb start line: 27 end line: 30 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 440 unit: sitemaps file: lib/crawler/robots_txt_parser.rb start line: 38 end line: 41 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 441 unit: allow_all? file: lib/crawler/robots_txt_parser.rb start line: 47 end line: 50 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 442 unit: allow_none? file: lib/crawler/robots_txt_parser.rb start line: 51 end line: 54 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 443 unit: allow_none_reason file: lib/crawler/robots_txt_parser.rb start line: 75 end line: 78 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 444 unit: initialize file: lib/crawler/logging/logger.rb start line: 16 end line: 20 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 445 unit: debug file: lib/crawler/logging/logger.rb start line: 27 end line: 30 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 446 unit: info file: lib/crawler/logging/logger.rb start line: 31 end line: 34 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 447 unit: warn file: lib/crawler/logging/logger.rb start line: 35 end line: 38 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 448 unit: error file: lib/crawler/logging/logger.rb start line: 39 end line: 42 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 449 unit: fatal file: lib/crawler/logging/logger.rb start line: 43 end line: 46 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 450 unit: add file: lib/crawler/logging/logger.rb start line: 47 end line: 50 size: 3 LOC McCabe index: 1 number of parameters: 2 id: 451 unit: << file: lib/crawler/logging/logger.rb start line: 51 end line: 54 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 452 unit: add_handler file: lib/crawler/logging/logger.rb start line: 55 end line: 58 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 453 unit: add_tags file: lib/crawler/logging/handler/file.rb start line: 46 end line: 50 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 454 unit: level file: lib/crawler/logging/handler/file.rb start line: 62 end line: 65 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 455 unit: log file: lib/crawler/logging/handler/base.rb start line: 19 end line: 22 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 456 unit: add_tags file: lib/crawler/logging/handler/base.rb start line: 23 end line: 26 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 457 unit: add_tags file: lib/crawler/logging/handler/stdout.rb start line: 44 end line: 48 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 458 unit: level file: lib/crawler/logging/handler/stdout.rb start line: 60 end line: 63 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 459 unit: run file: lib/crawler/executor.rb start line: 11 end line: 14 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 460 unit: http_client_status file: lib/crawler/executor.rb start line: 16 end line: 19 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 461 unit: self.transform file: lib/crawler/content_engine/transformer.rb start line: 15 end line: 18 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 462 unit: self.replace_with_whitespace? file: lib/crawler/content_engine/utils.rb start line: 86 end line: 89 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 463 unit: initialize file: lib/crawler/data/seen_urls.rb start line: 11 end line: 14 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 464 unit: clear file: lib/crawler/data/seen_urls.rb start line: 15 end line: 18 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 465 unit: count file: lib/crawler/data/seen_urls.rb start line: 19 end line: 22 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 466 unit: delete file: lib/crawler/data/seen_urls.rb start line: 23 end line: 26 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 467 unit: add? file: lib/crawler/data/seen_urls.rb start line: 35 end line: 38 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 468 unit: description file: lib/crawler/data/rule.rb start line: 39 end line: 42 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 469 unit: normalized_hash file: lib/crawler/data/url.rb start line: 21 end line: 24 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 470 unit: domain file: lib/crawler/data/url.rb start line: 38 end line: 41 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 471 unit: supported_scheme? file: lib/crawler/data/url.rb start line: 50 end line: 53 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 472 unit: path_segments_count file: lib/crawler/data/url.rb start line: 55 end line: 58 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 473 unit: params_count file: lib/crawler/data/url.rb start line: 60 end line: 63 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 474 unit: java_url file: lib/crawler/data/url.rb start line: 65 end line: 68 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 475 unit: to_s file: lib/crawler/data/link.rb start line: 34 end line: 37 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 476 unit: hash file: lib/crawler/data/link.rb start line: 41 end line: 44 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 477 unit: == file: lib/crawler/data/link.rb start line: 45 end line: 48 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 478 unit: === file: lib/crawler/data/link.rb start line: 49 end line: 52 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 479 unit: eql? file: lib/crawler/data/link.rb start line: 53 end line: 56 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 480 unit: rel file: lib/crawler/data/link.rb start line: 92 end line: 95 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 481 unit: rel_nofollow? file: lib/crawler/data/link.rb start line: 97 end line: 100 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 482 unit: initialize file: lib/crawler/data/crawl_result/html.rb start line: 18 end line: 21 size: 3 LOC McCabe index: 1 number of parameters: 2 id: 483 unit: parsed_content file: lib/crawler/data/crawl_result/html.rb start line: 22 end line: 25 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 484 unit: to_s file: lib/crawler/data/crawl_result/html.rb start line: 26 end line: 29 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 485 unit: canonical_url file: lib/crawler/data/crawl_result/html.rb start line: 91 end line: 94 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 486 unit: meta_nofollow? file: lib/crawler/data/crawl_result/html.rb start line: 102 end line: 105 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 487 unit: meta_noindex? file: lib/crawler/data/crawl_result/html.rb start line: 107 end line: 110 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 488 unit: extract_attribute_value file: lib/crawler/data/crawl_result/html.rb start line: 202 end line: 205 size: 3 LOC McCabe index: 1 number of parameters: 2 id: 489 unit: content_hash file: lib/crawler/data/crawl_result/content_extractable_file.rb start line: 28 end line: 31 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 490 unit: base64_encoded_content file: lib/crawler/data/crawl_result/content_extractable_file.rb start line: 32 end line: 35 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 491 unit: file_name file: lib/crawler/data/crawl_result/content_extractable_file.rb start line: 36 end line: 39 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 492 unit: url_hash file: lib/crawler/data/crawl_result/base.rb start line: 38 end line: 41 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 493 unit: site_url file: lib/crawler/data/crawl_result/base.rb start line: 42 end line: 45 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 494 unit: to_json file: lib/crawler/data/crawl_result/base.rb start line: 56 end line: 59 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 495 unit: to_s file: lib/crawler/data/crawl_result/base.rb start line: 60 end line: 63 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 496 unit: inspect file: lib/crawler/data/crawl_result/base.rb start line: 64 end line: 67 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 497 unit: error? file: lib/crawler/data/crawl_result/base.rb start line: 69 end line: 72 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 498 unit: unsupported_content_type? file: lib/crawler/data/crawl_result/base.rb start line: 73 end line: 76 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 499 unit: success? file: lib/crawler/data/crawl_result/base.rb start line: 77 end line: 80 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 500 unit: sitemap? file: lib/crawler/data/crawl_result/base.rb start line: 81 end line: 84 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 501 unit: html? file: lib/crawler/data/crawl_result/base.rb start line: 85 end line: 88 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 502 unit: content_extractable_file? file: lib/crawler/data/crawl_result/base.rb start line: 89 end line: 92 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 503 unit: redirect? file: lib/crawler/data/crawl_result/base.rb start line: 93 end line: 96 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 504 unit: to_h file: lib/crawler/data/crawl_result/error.rb start line: 31 end line: 34 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 505 unit: to_s file: lib/crawler/data/crawl_result/error.rb start line: 35 end line: 38 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 506 unit: to_s file: lib/crawler/data/crawl_result/redirect.rb start line: 43 end line: 46 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 507 unit: original_url file: lib/crawler/data/crawl_result/redirect.rb start line: 47 end line: 50 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 508 unit: redirect_count file: lib/crawler/data/crawl_result/redirect.rb start line: 51 end line: 54 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 509 unit: always_allowed file: lib/crawler/data/rule_engine_outcome.rb start line: 12 end line: 15 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 510 unit: error_denied file: lib/crawler/data/rule_engine_outcome.rb start line: 16 end line: 19 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 511 unit: too_many_redirects file: lib/crawler/data/rule_engine_outcome.rb start line: 20 end line: 23 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 512 unit: content_type_denied file: lib/crawler/data/rule_engine_outcome.rb start line: 24 end line: 27 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 513 unit: purge_crawl_allowed file: lib/crawler/data/rule_engine_outcome.rb start line: 28 end line: 31 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 514 unit: robots_txt_disallowed file: lib/crawler/data/rule_engine_outcome.rb start line: 46 end line: 49 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 515 unit: denied? file: lib/crawler/data/rule_engine_outcome.rb start line: 75 end line: 78 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 516 unit: allowed? file: lib/crawler/data/rule_engine_outcome.rb start line: 79 end line: 82 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 517 unit: denied? file: lib/crawler/data/rule_engine_outcome.rb start line: 93 end line: 96 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 518 unit: allowed? file: lib/crawler/data/rule_engine_outcome.rb start line: 97 end line: 100 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 519 unit: denied? file: lib/crawler/data/rule_engine_outcome.rb start line: 108 end line: 111 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 520 unit: allowed? file: lib/crawler/data/rule_engine_outcome.rb start line: 112 end line: 115 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 521 unit: check_queue_size! file: lib/crawler/data/url_queue/base.rb start line: 32 end line: 35 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 522 unit: warning_threshold file: lib/crawler/data/url_queue/base.rb start line: 38 end line: 41 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 523 unit: fetch file: lib/crawler/data/url_queue/base.rb start line: 68 end line: 71 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 524 unit: clear file: lib/crawler/data/url_queue/base.rb start line: 73 end line: 76 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 525 unit: delete file: lib/crawler/data/url_queue/base.rb start line: 78 end line: 81 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 526 unit: length file: lib/crawler/data/url_queue/base.rb start line: 89 end line: 92 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 527 unit: empty? file: lib/crawler/data/url_queue/base.rb start line: 93 end line: 96 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 528 unit: any? file: lib/crawler/data/url_queue/base.rb start line: 97 end line: 100 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 529 unit: fetch file: lib/crawler/data/url_queue/memory_only.rb start line: 73 end line: 76 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 530 unit: raw_url file: lib/crawler/data/domain.rb start line: 20 end line: 23 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 531 unit: robots_txt_url file: lib/crawler/data/domain.rb start line: 24 end line: 27 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 532 unit: == file: lib/crawler/data/domain.rb start line: 35 end line: 38 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 533 unit: to_s file: lib/crawler/data/domain.rb start line: 39 end line: 42 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 534 unit: inspect file: lib/crawler/data/crawl_task.rb start line: 27 end line: 30 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 535 unit: auth_type file: lib/crawler/data/crawl_task.rb start line: 31 end line: 34 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 536 unit: http_url_with_auth? file: lib/crawler/data/crawl_task.rb start line: 35 end line: 38 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 537 unit: == file: lib/crawler/data/crawl_task.rb start line: 39 end line: 42 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 538 unit: unique_id file: lib/crawler/data/crawl_task.rb start line: 56 end line: 59 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 539 unit: sitemap? file: lib/crawler/data/crawl_task.rb start line: 81 end line: 84 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 540 unit: content? file: lib/crawler/data/crawl_task.rb start line: 85 end line: 88 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 541 unit: robots_txt? file: lib/crawler/data/crawl_task.rb start line: 89 end line: 92 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 542 unit: denied_outcome file: lib/crawler/rule_engine/base.rb start line: 85 end line: 88 size: 3 LOC McCabe index: 1 number of parameters: 2 id: 543 unit: allowed_outcome file: lib/crawler/rule_engine/base.rb start line: 89 end line: 92 size: 3 LOC McCabe index: 1 number of parameters: 2 id: 544 unit: always_allow file: lib/crawler/robots_txt_service.rb start line: 13 end line: 16 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 545 unit: registered? file: lib/crawler/robots_txt_service.rb start line: 25 end line: 28 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 546 unit: parser_for_domain file: lib/crawler/robots_txt_service.rb start line: 38 end line: 41 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 547 unit: registered? file: lib/crawler/robots_txt_service.rb start line: 75 end line: 78 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 548 unit: register_crawl_result file: lib/crawler/robots_txt_service.rb start line: 79 end line: 82 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 549 unit: url_disallowed_outcome file: lib/crawler/robots_txt_service.rb start line: 83 end line: 86 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 550 unit: sitemaps file: lib/crawler/robots_txt_service.rb start line: 87 end line: 90 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 551 unit: allowed? file: lib/crawler/robots_txt_service.rb start line: 93 end line: 96 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 552 unit: initialize file: lib/crawler/document_mapper.rb start line: 14 end line: 17 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 553 unit: remove_empty_values file: lib/crawler/document_mapper.rb start line: 109 end line: 112 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 554 unit: call file: lib/crawler/cli/version.rb start line: 17 end line: 20 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 555 unit: initialize file: lib/crawler/mock_executor.rb start line: 15 end line: 18 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 556 unit: http_client_status file: lib/crawler/mock_executor.rb start line: 19 end line: 22 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 557 unit: head_request file: lib/crawler/http_executor.rb start line: 160 end line: 163 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 558 unit: get_request file: lib/crawler/http_executor.rb start line: 164 end line: 167 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 559 unit: content_extractable_file_mime_types file: lib/crawler/http_executor.rb start line: 353 end line: 356 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 560 unit: extractable_content file: lib/crawler/http_executor.rb start line: 358 end line: 361 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 561 unit: initialize file: lib/crawler/mock_event_logger.rb start line: 15 end line: 18 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 562 unit: self.monotonic_now file: lib/crawler/core_ext.rb start line: 17 end line: 20 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 563 unit: failure? file: lib/crawler/url_validator/result.rb start line: 19 end line: 22 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 564 unit: valid_checks file: lib/crawler/url_validator.rb start line: 87 end line: 90 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 565 unit: url file: lib/crawler/url_validator.rb start line: 108 end line: 111 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 566 unit: failed_checks file: lib/crawler/url_validator.rb start line: 118 end line: 121 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 567 unit: any_failed_results? file: lib/crawler/url_validator.rb start line: 124 end line: 127 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 568 unit: http_executor file: lib/crawler/url_validator.rb start line: 162 end line: 165 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 569 unit: proxy_configured? file: lib/crawler/url_validator.rb start line: 166 end line: 169 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 570 unit: crawler_api_config file: lib/crawler/url_validator.rb start line: 170 end line: 173 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 571 unit: http_proxy_host file: lib/crawler/http_utils/config.rb start line: 56 end line: 59 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 572 unit: http_proxy_port file: lib/crawler/http_utils/config.rb start line: 60 end line: 63 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 573 unit: http_proxy_username file: lib/crawler/http_utils/config.rb start line: 64 end line: 67 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 574 unit: http_proxy_password file: lib/crawler/http_utils/config.rb start line: 68 end line: 71 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 575 unit: http_proxy_scheme file: lib/crawler/http_utils/config.rb start line: 72 end line: 75 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 576 unit: ssl_ca_certificates file: lib/crawler/http_utils/config.rb start line: 76 end line: 79 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 577 unit: ssl_verification_mode file: lib/crawler/http_utils/config.rb start line: 80 end line: 83 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 578 unit: user_agent file: lib/crawler/http_utils/config.rb start line: 84 end line: 87 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 579 unit: loopback_allowed? file: lib/crawler/http_utils/config.rb start line: 88 end line: 91 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 580 unit: private_networks_allowed? file: lib/crawler/http_utils/config.rb start line: 92 end line: 95 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 581 unit: max_pool_size file: lib/crawler/http_utils/config.rb start line: 96 end line: 99 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 582 unit: pool_max_per_route file: lib/crawler/http_utils/config.rb start line: 100 end line: 103 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 583 unit: socket_timeout file: lib/crawler/http_utils/config.rb start line: 104 end line: 107 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 584 unit: connection_request_timeout file: lib/crawler/http_utils/config.rb start line: 108 end line: 111 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 585 unit: connect_timeout file: lib/crawler/http_utils/config.rb start line: 112 end line: 115 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 586 unit: check_connection_timeout file: lib/crawler/http_utils/config.rb start line: 116 end line: 119 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 587 unit: compression_enabled file: lib/crawler/http_utils/config.rb start line: 120 end line: 123 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 588 unit: crawler_default file: lib/crawler/http_utils/config.rb start line: 126 end line: 129 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 589 unit: release_connection file: lib/crawler/http_utils/response.rb start line: 35 end line: 38 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 590 unit: apache_status_line file: lib/crawler/http_utils/response.rb start line: 57 end line: 60 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 591 unit: code file: lib/crawler/http_utils/response.rb start line: 61 end line: 64 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 592 unit: reason_phrase file: lib/crawler/http_utils/response.rb start line: 65 end line: 68 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 593 unit: content_length file: lib/crawler/http_utils/response.rb start line: 87 end line: 90 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 594 unit: content_type file: lib/crawler/http_utils/response.rb start line: 91 end line: 94 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 595 unit: mime_type file: lib/crawler/http_utils/response.rb start line: 95 end line: 98 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 596 unit: time_since_request_start file: lib/crawler/http_utils/response.rb start line: 99 end line: 102 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 597 unit: redirect? file: lib/crawler/http_utils/response.rb start line: 103 end line: 106 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 598 unit: error? file: lib/crawler/http_utils/response.rb start line: 107 end line: 110 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 599 unit: unsupported_method? file: lib/crawler/http_utils/response.rb start line: 111 end line: 114 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 600 unit: redirect_location file: lib/crawler/http_utils/response.rb start line: 115 end line: 118 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 601 unit: checkClientTrusted file: lib/crawler/http_utils/all_trusting_trust_manager.rb start line: 18 end line: 21 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 602 unit: checkServerTrusted file: lib/crawler/http_utils/all_trusting_trust_manager.rb start line: 22 end line: 25 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 603 unit: getAcceptedIssuers file: lib/crawler/http_utils/all_trusting_trust_manager.rb start line: 26 end line: 29 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 604 unit: loopback_allowed? file: lib/crawler/http_utils/filtering_dns_resolver.rb start line: 32 end line: 35 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 605 unit: private_networks_allowed? file: lib/crawler/http_utils/filtering_dns_resolver.rb start line: 36 end line: 39 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 606 unit: suggestion_message file: lib/crawler/http_utils/exceptions.rb start line: 12 end line: 15 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 607 unit: initialize file: lib/crawler/http_utils/exceptions.rb start line: 18 end line: 21 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 608 unit: new file: lib/crawler/http_utils/exceptions.rb start line: 29 end line: 32 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 609 unit: message file: lib/crawler/http_utils/exceptions.rb start line: 33 end line: 36 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 610 unit: supported_encodings file: lib/crawler/http_utils/exceptions.rb start line: 37 end line: 40 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 611 unit: self.exception_root_cause file: lib/crawler/http_utils/exceptions.rb start line: 55 end line: 58 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 612 unit: format_message file: lib/crawler/http_utils/exceptions.rb start line: 66 end line: 69 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 613 unit: potential_cause file: lib/crawler/http_utils/exceptions.rb start line: 82 end line: 85 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 614 unit: suggestion_message file: lib/crawler/http_utils/exceptions.rb start line: 86 end line: 89 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 615 unit: potential_cause file: lib/crawler/http_utils/exceptions.rb start line: 92 end line: 95 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 616 unit: error_message file: lib/crawler/http_utils/exceptions.rb start line: 124 end line: 127 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 617 unit: format_message file: lib/crawler/http_utils/exceptions.rb start line: 128 end line: 131 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 618 unit: error_message file: lib/crawler/http_utils/exceptions.rb start line: 135 end line: 138 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 619 unit: suggestion_message file: lib/crawler/http_utils/exceptions.rb start line: 139 end line: 142 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 620 unit: error_message file: lib/crawler/http_utils/exceptions.rb start line: 146 end line: 149 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 621 unit: suggestion_message file: lib/crawler/http_utils/exceptions.rb start line: 150 end line: 153 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 622 unit: error_message file: lib/crawler/http_utils/exceptions.rb start line: 157 end line: 160 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 623 unit: suggestion_message file: lib/crawler/http_utils/exceptions.rb start line: 161 end line: 164 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 624 unit: error_message file: lib/crawler/http_utils/exceptions.rb start line: 168 end line: 171 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 625 unit: suggestion_message file: lib/crawler/http_utils/exceptions.rb start line: 172 end line: 175 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 626 unit: error_message file: lib/crawler/http_utils/exceptions.rb start line: 179 end line: 182 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 627 unit: suggestion_message file: lib/crawler/http_utils/exceptions.rb start line: 183 end line: 186 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 628 unit: error_message file: lib/crawler/http_utils/exceptions.rb start line: 190 end line: 193 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 629 unit: configure_crawl_id! file: lib/crawler/api/config.rb start line: 265 end line: 268 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 630 unit: configure_robots_txt_service! file: lib/crawler/api/config.rb start line: 373 end line: 376 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 631 unit: configure_http_header_service! file: lib/crawler/api/config.rb start line: 377 end line: 380 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 632 unit: events file: lib/crawler/api/config.rb start line: 459 end line: 462 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 633 unit: stats file: lib/crawler/api/config.rb start line: 464 end line: 467 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 634 unit: document_mapper file: lib/crawler/api/config.rb start line: 468 end line: 471 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 635 unit: shutdown_started? file: lib/crawler/api/crawl.rb start line: 50 end line: 53 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 636 unit: allow_resume? file: lib/crawler/api/crawl.rb start line: 55 end line: 58 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 637 unit: coordinator file: lib/crawler/api/crawl.rb start line: 80 end line: 83 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 638 unit: log_exception file: lib/crawler/api/crawl.rb start line: 190 end line: 193 size: 3 LOC McCabe index: 1 number of parameters: 3 id: 639 unit: ingestion_stats file: lib/crawler/output_sink/elasticsearch.rb start line: 184 end line: 187 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 640 unit: es_config file: lib/crawler/output_sink/elasticsearch.rb start line: 196 end line: 199 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 641 unit: client file: lib/crawler/output_sink/elasticsearch.rb start line: 200 end line: 203 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 642 unit: index_name file: lib/crawler/output_sink/elasticsearch.rb start line: 204 end line: 207 size: 3 LOC McCabe index: 2 number of parameters: 0 id: 643 unit: pipeline_params file: lib/crawler/output_sink/elasticsearch.rb start line: 222 end line: 225 size: 3 LOC McCabe index: 3 number of parameters: 0 id: 644 unit: create_rule_engine file: lib/crawler/output_sink/base.rb start line: 22 end line: 25 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 645 unit: write file: lib/crawler/output_sink/base.rb start line: 26 end line: 29 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 646 unit: fetch_purge_docs file: lib/crawler/output_sink/base.rb start line: 30 end line: 33 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 647 unit: purge file: lib/crawler/output_sink/base.rb start line: 34 end line: 37 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 648 unit: to_doc file: lib/crawler/output_sink/base.rb start line: 38 end line: 41 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 649 unit: outcome file: lib/crawler/output_sink/base.rb start line: 54 end line: 57 size: 3 LOC McCabe index: 1 number of parameters: 2 id: 650 unit: success file: lib/crawler/output_sink/base.rb start line: 58 end line: 61 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 651 unit: failure file: lib/crawler/output_sink/base.rb start line: 62 end line: 65 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 652 unit: last_stats_dump file: lib/crawler/event_generator.rb start line: 28 end line: 31 size: 3 LOC McCabe index: 1 number of parameters: 0 id: 653 unit: crawl_status_for_system_log file: lib/crawler/event_generator.rb start line: 132 end line: 135 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 654 unit: log_url_event file: lib/crawler/event_generator.rb start line: 317 end line: 320 size: 3 LOC McCabe index: 1 number of parameters: 2 id: 655 unit: log_crawl_event file: lib/crawler/event_generator.rb start line: 321 end line: 324 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 656 unit: log_crawl_metric file: lib/crawler/event_generator.rb start line: 325 end line: 328 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 657 unit: log_event file: lib/crawler/event_generator.rb start line: 332 end line: 335 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 658 unit: log_metric file: lib/crawler/event_generator.rb start line: 336 end line: 339 size: 3 LOC McCabe index: 1 number of parameters: 1 id: 659 unit: save file: lib/crawler/data/seen_urls.rb start line: 28 end line: 31 size: 2 LOC McCabe index: 1 number of parameters: 0 id: 660 unit: save file: lib/crawler/data/url_queue/base.rb start line: 83 end line: 86 size: 2 LOC McCabe index: 1 number of parameters: 0 id: 661 unit: close file: lib/crawler/output_sink/base.rb start line: 42 end line: 46 size: 2 LOC McCabe index: 1 number of parameters: 0 id: 662 unit: flush file: lib/crawler/output_sink/base.rb start line: 47 end line: 51 size: 2 LOC McCabe index: 1 number of parameters: 0 id: 663 unit: write file: lib/crawler/output_sink/null.rb start line: 13 end line: 16 size: 2 LOC McCabe index: 1 number of parameters: 1