extract_and_enqueue_links

in lib/crawler/coordinator.rb [423:432]


    def extract_and_enqueue_links(crawl_task, crawl_result)
      return if crawl_result.error? || @crawl_stage == CRAWL_STAGE_PURGE

      crawl_task_progress(crawl_task, 'extracting links')
      return enqueue_redirect_link(crawl_task, crawl_result) if crawl_result.redirect?
      return extract_and_enqueue_html_links(crawl_task, crawl_result) if crawl_result.html?

      extract_and_enqueue_sitemap_links(crawl_task, crawl_result) if crawl_result.sitemap?
    end