generate_docs

in versioned_plugins.rb [89:223]


  def generate_docs
    puts "writing to #{logstash_docs_path}"
    repos = octo.org_repos("logstash-plugins")
    repos = repos.map {|repo| repo.name }.select {|repo| repo.match(plugin_regex) }
    repos = (repos - PLUGIN_SKIP_LIST).sort.uniq.map {|repo| "logstash-plugins/#{repo}"}
    repos = repos.concat(ADDITIONAL_ORG_PLUGINS)

    puts "found #{repos.size} repos"

    
    timestamp_reference = since || Time.strptime($TIMESTAMP_REFERENCE, "%a, %d %b %Y %H:%M:%S %Z")
    puts "Generating docs since #{timestamp_reference.inspect}"

    plugins_indexes_to_rebuild = Util::ThreadsafeWrapper.for(Set.new)
    package_indexes_to_rebuild = Util::ThreadsafeWrapper.for(Set.new)

    plugin_version_index = Util::ThreadsafeIndex.new { Util::ThreadsafeWrapper.for(Set.new) }
    plugin_names_by_type = Util::ThreadsafeIndex.new { Util::ThreadsafeWrapper.for(Set.new) }

    
    repos_requiring_rebuild = Util::ThreadsafeWrapper.for(Set.new)

    
    repositories = repos.map do |repo_name|
      $stderr.puts("[#{repo_name}]: loading releases...")
      source = Source::Github.new(repo: repo_name, octokit: @octo)
      Repository::from_source(source.repo, source)
    end

    
    
    
    
    repositories.peach(parallelism) do |repository|
      latest_release = repository.last_release
      if latest_release.nil?
        $stderr.puts("#{repository.desc}: no releases on rubygems.\n")
        next
      end

      
      
      if latest_release.release_date.nil? || latest_release.release_date < timestamp_reference
        $stderr.puts("#{repository.desc}: no new releases.\n")
        next
      end

      
      
      $stderr.puts("#{repository.desc}: found new release\n")
      
          

      
      
      
      latest_release.with_embedded_plugins.each do |plugin|
        repos_requiring_rebuild.add(plugin.canonical_name) &&
            $stderr.puts("#{plugin.desc}: marking for reindex\n")
      end

    end

    
    repositories.peach(parallelism) do |repository|
      unless repos_requiring_rebuild.include?(repository.name)
        $stderr.puts("[repo:#{repository.name}]: rebuild not required. skipping.\n")
        latest_release = repository.last_release
        latest_release && latest_release.with_embedded_plugins.each do |plugin|
          next unless versions_index_exists?(plugin.name, plugin.type)
          plugin_names_by_type.fetch(plugin.type).add(plugin.name)
        end
        next
      end

      $stderr.puts("[repo:#{repository.name}]: rebuilding versioned docs\n")
      repository.source_tagged_releases.each do |released_plugin|
        released_plugin.with_embedded_plugins.each do |plugin|
          if expand_plugin_doc(plugin)
            plugins_indexes_to_rebuild.add(plugin.canonical_name)
            plugin_version_index.fetch(plugin.canonical_name).add(plugin)
            plugin_names_by_type.fetch(plugin.type).add(plugin.name)
          else
            $stderr.puts("#{plugin.desc}: documentation not available; skipping remaining releases from repository\n")
            break false
          end
        end || break

        break if latest_only?
      end
    end

    $stderr.puts("REINDEXING PLUGINS, loading plugin aliases...")
    alias_definitions_by_type = Util::AliasDefinitionsLoader.get_alias_definitions

    
    alias_definitions_by_type.each do |type, alias_definitions|
      alias_definitions.each do |alias_definition|
        plugin_names_by_type.fetch(type).add(alias_definition.fetch("alias"))
      end
    end

    
    $stderr.puts("REINDEXING PLUGINS ALIASES... #{alias_definitions_by_type.size}\n")
    alias_definitions_by_type.each do |type, alias_definitions|
      alias_definitions.each do |alias_definition|
        $stderr.puts("[plugin:#{alias_definition.fetch("alias")}] reindexing\n")
        write_alias_index(type, alias_definition.fetch("alias"), alias_definition.fetch("from"))
      end
    end

    
    $stderr.puts("REINDEXING PLUGINS... #{plugins_indexes_to_rebuild.size}\n")
    plugins_indexes_to_rebuild.each do |canonical_name|
      $stderr.puts("[plugin:#{canonical_name}] reindexing\n")
      versions = plugin_version_index.fetch(canonical_name).sort_by(&:version).reverse.map do |plugin|
        [plugin.tag, plugin.release_date.strftime("%Y-%m-%d")]
      end
      _, type, name = canonical_name.split('-',3)
      write_versions_index(name, type, versions)
    end

    
    package_indexes_to_rebuild.each do |canonical_name|
      
    end

    
    $stderr.puts("REINDEXING TYPES... #{}\n")
    plugin_names_by_type.each do |type, names|
      $stderr.puts("[type:#{type}] reindexing\n")
      write_type_index(type, names.sort)
    end
  end