in versioned_plugins.rb [89:223]
def generate_docs
puts "writing to #{logstash_docs_path}"
repos = octo.org_repos("logstash-plugins")
repos = repos.map {|repo| repo.name }.select {|repo| repo.match(plugin_regex) }
repos = (repos - PLUGIN_SKIP_LIST).sort.uniq.map {|repo| "logstash-plugins/#{repo}"}
repos = repos.concat(ADDITIONAL_ORG_PLUGINS)
puts "found #{repos.size} repos"
timestamp_reference = since || Time.strptime($TIMESTAMP_REFERENCE, "%a, %d %b %Y %H:%M:%S %Z")
puts "Generating docs since #{timestamp_reference.inspect}"
plugins_indexes_to_rebuild = Util::ThreadsafeWrapper.for(Set.new)
package_indexes_to_rebuild = Util::ThreadsafeWrapper.for(Set.new)
plugin_version_index = Util::ThreadsafeIndex.new { Util::ThreadsafeWrapper.for(Set.new) }
plugin_names_by_type = Util::ThreadsafeIndex.new { Util::ThreadsafeWrapper.for(Set.new) }
repos_requiring_rebuild = Util::ThreadsafeWrapper.for(Set.new)
repositories = repos.map do |repo_name|
$stderr.puts("[#{repo_name}]: loading releases...")
source = Source::Github.new(repo: repo_name, octokit: @octo)
Repository::from_source(source.repo, source)
end
repositories.peach(parallelism) do |repository|
latest_release = repository.last_release
if latest_release.nil?
$stderr.puts("#{repository.desc}: no releases on rubygems.\n")
next
end
if latest_release.release_date.nil? || latest_release.release_date < timestamp_reference
$stderr.puts("#{repository.desc}: no new releases.\n")
next
end
$stderr.puts("#{repository.desc}: found new release\n")
latest_release.with_embedded_plugins.each do |plugin|
repos_requiring_rebuild.add(plugin.canonical_name) &&
$stderr.puts("#{plugin.desc}: marking for reindex\n")
end
end
repositories.peach(parallelism) do |repository|
unless repos_requiring_rebuild.include?(repository.name)
$stderr.puts("[repo:#{repository.name}]: rebuild not required. skipping.\n")
latest_release = repository.last_release
latest_release && latest_release.with_embedded_plugins.each do |plugin|
next unless versions_index_exists?(plugin.name, plugin.type)
plugin_names_by_type.fetch(plugin.type).add(plugin.name)
end
next
end
$stderr.puts("[repo:#{repository.name}]: rebuilding versioned docs\n")
repository.source_tagged_releases.each do |released_plugin|
released_plugin.with_embedded_plugins.each do |plugin|
if expand_plugin_doc(plugin)
plugins_indexes_to_rebuild.add(plugin.canonical_name)
plugin_version_index.fetch(plugin.canonical_name).add(plugin)
plugin_names_by_type.fetch(plugin.type).add(plugin.name)
else
$stderr.puts("#{plugin.desc}: documentation not available; skipping remaining releases from repository\n")
break false
end
end || break
break if latest_only?
end
end
$stderr.puts("REINDEXING PLUGINS, loading plugin aliases...")
alias_definitions_by_type = Util::AliasDefinitionsLoader.get_alias_definitions
alias_definitions_by_type.each do |type, alias_definitions|
alias_definitions.each do |alias_definition|
plugin_names_by_type.fetch(type).add(alias_definition.fetch("alias"))
end
end
$stderr.puts("REINDEXING PLUGINS ALIASES... #{alias_definitions_by_type.size}\n")
alias_definitions_by_type.each do |type, alias_definitions|
alias_definitions.each do |alias_definition|
$stderr.puts("[plugin:#{alias_definition.fetch("alias")}] reindexing\n")
write_alias_index(type, alias_definition.fetch("alias"), alias_definition.fetch("from"))
end
end
$stderr.puts("REINDEXING PLUGINS... #{plugins_indexes_to_rebuild.size}\n")
plugins_indexes_to_rebuild.each do |canonical_name|
$stderr.puts("[plugin:#{canonical_name}] reindexing\n")
versions = plugin_version_index.fetch(canonical_name).sort_by(&:version).reverse.map do |plugin|
[plugin.tag, plugin.release_date.strftime("%Y-%m-%d")]
end
_, type, name = canonical_name.split('-',3)
write_versions_index(name, type, versions)
end
package_indexes_to_rebuild.each do |canonical_name|
end
$stderr.puts("REINDEXING TYPES... #{}\n")
plugin_names_by_type.each do |type, names|
$stderr.puts("[type:#{type}] reindexing\n")
write_type_index(type, names.sort)
end
end