in lib/crawler/api/crawl.rb [117:137]
def start_url_test!(endpoint)
events.crawl_start(
url_queue_items: crawl_queue.length,
seen_urls: seen_urls.count
)
@sink = Crawler::OutputSink::File.new(config)
coordinator.run_urltest_crawl!(endpoint)
record_overall_outcome(coordinator.crawl_results)
print_url_test_results(coordinator.url_test_results)
rescue StandardError => e
log_exception(e, 'Unexpected error while running the crawl')
record_outcome(
outcome: :failure,
message: 'Unexpected error while running the crawl, check system logs for details'
)
ensure
crawl_queue.delete
seen_urls.clear
end