in lib/crawler/http_utils/filtering_dns_resolver.rb [40:58]
def remove_private_addresses(host, resolved_addresses)
return resolved_addresses if resolved_addresses.blank?
valid_addresses, invalid_addresses = resolved_addresses.partition do |a|
allowed_address?(a)
end
if invalid_addresses.present?
@logger.info("Rejected invalid addresses #{invalid_addresses.map(&:host_address)} for host #{host.inspect}")
end
if valid_addresses.empty?
error = "Unable to request #{host.inspect} because it resolved to only private/invalid addresses"
raise Crawler::HttpUtils::InvalidHost, error
end
valid_addresses
end