in lib/crawler/http_executor.rb [104:134]
def handling_http_errors(crawl_task)
yield
rescue Crawler::HttpUtils::ResponseTooLarge => e
logger.warn(e.message)
Crawler::Data::CrawlResult::Error.new(
url: crawl_task.url,
error: e.message
)
rescue Crawler::HttpUtils::ConnectTimeout => e
timeout_error(crawl_task:, exception: e, error: 'connection_timeout')
rescue Crawler::HttpUtils::SocketTimeout => e
timeout_error(crawl_task:, exception: e, error: 'read_timeout')
rescue Crawler::HttpUtils::RequestTimeout => e
timeout_error(crawl_task:, exception: e, error: e.message)
rescue Crawler::HttpUtils::SslException => e
logger.error("SSL error while performing HTTP request: #{e.message}. #{e.suggestion_message}")
Crawler::Data::CrawlResult::Error.new(
url: crawl_task.url,
error: e.message,
suggestion_message: e.suggestion_message
)
rescue Crawler::HttpUtils::BaseError => e
error = "Failed HTTP request: #{e}. #{e.suggestion_message}"
logger.error(error)
Crawler::Data::CrawlResult::Error.new(
url: crawl_task.url,
error:,
suggestion_message: e.suggestion_message
)
end