remove_private_addresses

in lib/crawler/http_utils/filtering_dns_resolver.rb [40:58]


      def remove_private_addresses(host, resolved_addresses)
        return resolved_addresses if resolved_addresses.blank?

        valid_addresses, invalid_addresses = resolved_addresses.partition do |a|
          allowed_address?(a)
        end

        if invalid_addresses.present?
          @logger.info("Rejected invalid addresses #{invalid_addresses.map(&:host_address)} for host #{host.inspect}")
        end

        if valid_addresses.empty?
          error = "Unable to request #{host.inspect} because it resolved to only private/invalid addresses"
          raise Crawler::HttpUtils::InvalidHost, error
        end

        valid_addresses
      end