handling_http_errors

in lib/crawler/http_executor.rb [104:134]


    def handling_http_errors(crawl_task)
      yield
    rescue Crawler::HttpUtils::ResponseTooLarge => e
      logger.warn(e.message)
      Crawler::Data::CrawlResult::Error.new(
        url: crawl_task.url,
        error: e.message
      )
    rescue Crawler::HttpUtils::ConnectTimeout => e
      timeout_error(crawl_task:, exception: e, error: 'connection_timeout')
    rescue Crawler::HttpUtils::SocketTimeout => e
      timeout_error(crawl_task:, exception: e, error: 'read_timeout')
    rescue Crawler::HttpUtils::RequestTimeout => e
      timeout_error(crawl_task:, exception: e, error: e.message)
    rescue Crawler::HttpUtils::SslException => e
      logger.error("SSL error while performing HTTP request: #{e.message}. #{e.suggestion_message}")
      Crawler::Data::CrawlResult::Error.new(
        url: crawl_task.url,
        error: e.message,
        suggestion_message: e.suggestion_message
      )
    rescue Crawler::HttpUtils::BaseError => e
      error = "Failed HTTP request: #{e}. #{e.suggestion_message}"
      logger.error(error)
      Crawler::Data::CrawlResult::Error.new(
        url: crawl_task.url,
        error:,
        suggestion_message: e.suggestion_message
      )
    end