in lib/crawler/coordinator.rb [423:432]
def extract_and_enqueue_links(crawl_task, crawl_result)
return if crawl_result.error? || @crawl_stage == CRAWL_STAGE_PURGE
crawl_task_progress(crawl_task, 'extracting links')
return enqueue_redirect_link(crawl_task, crawl_result) if crawl_result.redirect?
return extract_and_enqueue_html_links(crawl_task, crawl_result) if crawl_result.html?
extract_and_enqueue_sitemap_links(crawl_task, crawl_result) if crawl_result.sitemap?
end