url_fetch

in lib/crawler/event_generator.rb [156:185]

• 26 lines of code
• 4 McCabe index (conditional complexity)

    def url_fetch(url:, crawl_result:, auth_type: nil) 
      status_code = crawl_result.status_code
      outcome = outcome_from_status_code(status_code)
      system_logger.debug("Fetched a page '#{url}' with a status code #{status_code} and an outcome of '#{outcome}'")

      event = {
        'crawler.url.auth.type' => auth_type,
        'event.type' => 'access',
        'event.action' => 'url-fetch',
        'event.outcome' => outcome,
        'event.start' => crawl_result.start_time,
        'event.end' => crawl_result.end_time,
        'event.duration' => crawl_result.duration,
        'http.request.method' => 'get',
        'http.response.status_code' => status_code.to_s
      }

      if crawl_result.error?
        event['message'] = crawl_result.error
      elsif crawl_result.success?
        event['http.response.body.bytes'] = crawl_result.content.bytesize
      elsif crawl_result.redirect?
        event['crawler.url.redirect.location'] = crawl_result.location.to_s
        event['crawler.url.redirect.chain'] = crawl_result.redirect_chain.map(&:to_s)
        event['crawler.url.redirect.count'] = crawl_result.redirect_count
      end

      log_url_event(url, event)
    end