in lib/crawler/rule_engine/base.rb [64:82]
def output_crawl_result_outcome(crawl_result)
unless crawl_result.is_a?(Crawler::Data::CrawlResult::Base)
raise ArgumentError,
'Needs a Crawler::Data::CrawlResult::Base object'
end
return denied_outcome(:content_type_denied, crawl_result.error) if crawl_result.unsupported_content_type?
return denied_outcome(:error_denied, crawl_result.error) if crawl_result.error?
return denied_outcome(:noindex_meta_denied) if crawl_result.html? && crawl_result.meta_noindex?
if crawl_result.redirect? && crawl_result.redirect_count > config.max_redirects
error = "Too many redirects (#{crawl_result.redirect_count}) " \
"while trying to download the page at #{crawl_result.original_url.inspect}"
return denied_outcome(:too_many_redirects, error)
end
allowed_outcome
end