rake_tasks/doc_generator.rake (180 lines of code) (raw):

# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. require 'json' require 'fileutils' require 'logger' SRC_FILE = "#{__dir__}/docs/parsed_alternative_report.json".freeze EXAMPLES_TO_PARSE = JSON.parse(File.read("#{__dir__}/docs/examples_to_parse.json")).freeze TARGET_DIR = "#{__dir__}/../docs/examples/guide".freeze namespace :docs do desc 'Generate doc examples' task :generate do # Remove existing documents to avoid having outdated files FileUtils.remove_dir(TARGET_DIR) Dir.mkdir(TARGET_DIR) Dir['log/*.log'].each { |f| File.delete(f) } entries = json_data.select { |d| d['lang'] == 'console' } start_time = Time.now.to_i entries.each_with_index do |entry, index| percentage = index * 100 / entries.length hourglass = index.even? ? '⌛ ' : '⏳ ' # print "\r" + ("\e[A\e[K" * 2) if index > 0 puts "\e[H\e[2J" puts "📝 Generating file #{index + 1} of #{entries.length} - #{percentage}% complete" puts hourglass + '▩' * (percentage / 2) + '⬚' * (50 - percentage / 2) + ' ' + hourglass generate_docs(entry) end puts "Finished generating #{entries.length} files in #{Time.now.to_i - start_time} seconds" delete_first_log_line end desc 'Update report' task :update, [:branch] do |_, args| require 'elastic-transport' github_token = File.read(File.expand_path("~/.elastic/github.token")) transport_options = { headers: { Accept: 'application/vnd.github.v3.raw', Authorization: "token #{github_token}" } } client = Elastic::Transport::Client.new( host: 'https://api.github.com/', transport_options:transport_options ) path = '/repos/elastic/clients-flight-recorder/contents/recordings/docs/parsed-alternative-report.json' path = "#{path}?ref=#{args[:branch]}" if args[:branch] params = {} response = client.perform_request('GET', path, params) File.write(File.expand_path('./docs/parsed_alternative_report.json', __dir__), response.body) puts "Downloaded report for #{args[:branch] ? args[:branch] : 'main' } branch" end desc 'Add files from 200-ok log' task :add_files do `cat log/200-ok.log | xargs git add` end def json_data JSON.parse(File.read(SRC_FILE)) end def generate_docs(entry) require 'elasticsearch' filename = File.expand_path("#{TARGET_DIR}/#{entry['digest']}.asciidoc") unless entry['parsed_source'].empty? code = build_client_query(entry) TestDocs::perform(code, filename) write_file(code, filename) end end def self.build_client_query(entry) client_query = [] entry['parsed_source'].each do |entry| api = entry['api'] request_body = [] query = entry&.[]('query') params = entry&.[]('params') params = params&.merge(query) || query if query request_body << show_parameters(params) if params body = entry&.[]('body') request_body << show_body(body) if body request_body = request_body.compact.join(",\n").gsub('null', 'nil') code = if api.include? '_internal' "response = client.perform_request('#{entry['method']}', '#{api}', #{request_body})" else "response = client.#{api}(\n#{request_body}\n)\nputs response" end client_query << format_code(code) end client_query.join("\n\n") end def self.format_code(code) # Print code to the file File.open('temp.rb', 'w') do |f| f.puts code end # Format code with Rubocop require 'rubocop' options = "--config #{__dir__}/docs_rubocop_config.yml -o /dev/null -a ./temp.rb".split cli = RuboCop::CLI.new cli.run(options) # Read it back template = File.read('./temp.rb') File.delete('./temp.rb') # TODO: Manually remove final blank line since Rubocop is ignoring the config directive template.gsub(/\s+$/, '') end def show_parameters(params) param_string = [] params.each do |k, v| value = (is_number?(v) || is_boolean?(v)) ? v : "'#{v}'" param_string << "#{k.gsub('\"','')}: #{value}" end param_string.join(",\n\s\s") end def show_body(body) 'body: ' + JSON.pretty_generate(body) .gsub(/\"([a-z_]+)\":/,'\\1: ') # Use Ruby 2 hash syntax .gsub(/"([a-z_.]+\.[a-z_]+)+":/, '"\\1" =>') .gsub('aggs', 'aggregations') # Replace 'aggs' with 'aggregations' for consistency end def is_number?(value) Float(value) || Integer(value) rescue false end def is_boolean?(value) (['false', 'true'].include? value) || value.is_a?(TrueClass) || value.is_a?(FalseClass) end def write_file(code, filename) File.open(filename, 'w') do |f| f.puts <<~SRC [source, ruby] ---- #{code} ---- SRC end end def delete_first_log_line logfile = File.expand_path(__dir__ + '/../log/200-ok.log') content = IO.readlines(logfile, chomp: true) puts content.shift File.write(logfile, content.first) end end # # Test module to run the generated code # module TestDocs @formatter = -> (_, d, _, msg) { "[#{d}] : #{msg}" } def self.perform(code, filename) # Eval the example code, but remove printing out the response response = eval(code.gsub('puts response', '')) log_successful_code(filename) if response_successful(response) rescue Elastic::Transport::Transport::Errors::NotFound => e log_successful_code(filename) rescue Elastic::Transport::Transport::Errors::BadRequest => e if e.message.match? /resource_already_exists/ log_successful_code(filename) else log_elasticsearch_error(filename, e) end rescue Elastic::Transport::Transport::Error => e log_elasticsearch_error(filename, e) rescue ArgumentError, NoMethodError, TypeError => e logger = Logger.new('log/docs-generation-client.log') logger.formatter = @formatter logger.info("Located in #{filename}: #{e.message}\n") end def self.client @client ||= Elasticsearch::Client.new(trace: false, log: false) end def self.log_successful_code(filename) FileUtils.mkdir_p('./log') unless File.directory?('./log') logger = Logger.new('./log/200-ok.log') logger.formatter = -> (_, _, _, msg) { "#{msg} " } logger.info(filename) end def self.log_elasticsearch_error(filename, e) logger = Logger.new('log/docs-generation-elasticsearch.log') logger.formatter = @formatter logger.info("Located in #{filename}: #{e.message}\n") end def self.response_successful(response) [true, false].include?(response) || ( response.is_a?(Elasticsearch::API::Response) && [200, 201].include?(response.status) ) end end