utils/generate-examples.py

#!/usr/bin/env python
#  Licensed to Elasticsearch B.V. under one or more contributor
#  license agreements. See the NOTICE file distributed with
#  this work for additional information regarding copyright
#  ownership. Elasticsearch B.V. licenses this file to you under
#  the Apache License, Version 2.0 (the "License"); you may
#  not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
# 	http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.


import collections
import json
import os
import tempfile
from pathlib import Path

import black
from click.testing import CliRunner
from jinja2 import Environment, FileSystemLoader

code_root = Path(__file__).absolute().parent.parent
asciidocs_dir = code_root / "docs/examples"
flight_recorder_dir = code_root.parent / "clients-flight-recorder"
report_path = flight_recorder_dir / "recordings/docs/parsed-alternative-report.json"
substitutions = {"type": "doc_type", "from": "from_"}

jinja_env = Environment(
    loader=FileSystemLoader([code_root / "utils" / "templates"]),
    trim_blocks=True,
    lstrip_blocks=True,
)

files_to_generate = [
    "search/request-body.asciidoc",
    "mapping.asciidoc",
    "query-dsl.asciidoc",
    "query-dsl/query-string-query.asciidoc",
    "getting-started.asciidoc",
    "query-dsl/query_filter_context.asciidoc",
    "query-dsl/bool-query.asciidoc",
    "query-dsl/match-query.asciidoc",
    "indices/create-index.asciidoc",
    "docs/index_.asciidoc",
    "aggregations/bucket/terms-aggregation.asciidoc",
    "query-dsl/range-query.asciidoc",
    "search/search.asciidoc",
    "query-dsl/multi-match-query.asciidoc",
    "docs/bulk.asciidoc",
    "indices/delete-index.asciidoc",
    "indices/put-mapping.asciidoc",
    "docs/reindex.asciidoc",
    "query-dsl/term-query.asciidoc",
    "indices/templates.asciidoc",
    "getting-started.asciidoc",
    "docs/update.asciidoc",
    "query-dsl/match-all-query.asciidoc",
    "docs/get.asciidoc",
    "query-dsl/wildcard-query.asciidoc",
    "query-dsl/exists-query.asciidoc",
    "docs/delete-by-query.asciidoc",
    "mapping/params/format.asciidoc",
    "mapping/types/nested.asciidoc",
    "query-dsl/terms-query.asciidoc",
    "search/request/sort.asciidoc",
    "mapping/types/date.asciidoc",
    "indices/update-settings.asciidoc",
    "indices/aliases.asciidoc",
    "setup/install/check-running.asciidoc",
    "query-dsl/regexp-query.asciidoc",
    "query-dsl/function-score-query.asciidoc",
    "search/request/from-size.asciidoc",
    "cluster/health.asciidoc",
    "query-dsl/nested-query.asciidoc",
    "mapping/types/array.asciidoc",
    "mapping/params/fielddata.asciidoc",
    "search/count.asciidoc",
    "mapping/types/keyword.asciidoc",
    "docs/update-by-query.asciidoc",
    "search/suggesters.asciidoc",
    "api-conventions.asciidoc",
    "cat/indices.asciidoc",
    "query-dsl/match-phrase-query.asciidoc",
    "indices/get-index.asciidoc",
    "setup/logging-config.asciidoc",
    "docs/delete.asciidoc",
    "aggregations/metrics/valuecount-aggregation.asciidoc",
    "indices/get-mapping.asciidoc",
    "aggregations/bucket/filter-aggregation.asciidoc",
    "aggregations/bucket/datehistogram-aggregation.asciidoc",
    "mapping/types/numeric.asciidoc",
    "search/request/scroll.asciidoc",
    "mapping/fields/id-field.asciidoc",
    "search.asciidoc",
    "mapping/params/multi-fields.asciidoc",
    "cluster/allocation-explain.asciidoc",
    "cluster/get-settings.asciidoc",
    "cluster/update-settings.asciidoc",
    "health/health.asciidoc",
    "cluster/reroute.asciidoc",
    "inference/get-inference.asciidoc",
    "inference/delete-inference.asciidoc",
    "inference/post-inference.asciidoc",
    "inference/put-inference.asciidoc",
    "ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc",
    "ml/trained-models/apis/delete-trained-models-aliases.asciidoc",
    "ml/trained-models/apis/delete-trained-models.asciidoc",
    "ml/trained-models/apis/get-trained-models-stats.asciidoc",
    "ml/trained-models/apis/get-trained-models.asciidoc",
    "ml/trained-models/apis/infer-trained-model-deployment.asciidoc",
    "ml/trained-models/apis/infer-trained-model.asciidoc",
    "ml/trained-models/apis/put-trained-model-definition-part.asciidoc",
    "ml/trained-models/apis/put-trained-model-vocabulary.asciidoc",
    "ml/trained-models/apis/put-trained-models-aliases.asciidoc",
    "ml/trained-models/apis/put-trained-models.asciidoc",
    "ml/trained-models/apis/start-trained-model-deployment.asciidoc",
    "ml/trained-models/apis/stop-trained-model-deployment.asciidoc",
    "ml/trained-models/apis/update-trained-model-deployment.asciidoc",
    "setup/run-elasticsearch-locally.asciidoc",
    "setup/important-settings.asciidoc",
    "setup/secure-settings.asciidoc",
    "modules/cluster.asciidoc",
    "modules/cluster/misc.asciidoc",
    "modules/network.asciidoc",
    "modules/indices/request_cache.asciidoc",
    "setup/advanced-configuration.asciidoc",
    "setup/sysconfig/swap.asciidoc",
    "setup/sysconfig/file-descriptors.asciidoc",
    "modules/discovery/voting.asciidoc",
    "setup/add-nodes.asciidoc",
    "setup/restart-cluster.asciidoc",
    "modules/cluster/remote-clusters-api-key.asciidoc",
    "modules/cluster/remote-clusters-cert.asciidoc",
    "modules/discovery/voting.asciidoc",
    "modules/cluster/remote-clusters-migration.asciidoc",
    "modules/cluster/remote-clusters-troubleshooting.asciidoc",
    "upgrade/archived-settings.asciidoc",
    "index-modules/allocation/filtering.asciidoc",
    "index-modules/allocation/delayed.asciidoc",
    "index-modules/allocation/prioritization.asciidoc",
    "index-modules/allocation/total_shards.asciidoc",
    "index-modules/allocation/data_tier_allocation.asciidoc",
    "index-modules/blocks.asciidoc",
    "index-modules/similarity.asciidoc",
    "index-modules/slowlog.asciidoc",
    "index-modules/store.asciidoc",
    "index-modules/index-sorting.asciidoc",
    "index-modules/indexing-pressure.asciidoc",
    "mapping/dynamic-mapping.asciidoc",
    "mapping/dynamic/field-mapping.asciidoc",
    "mapping/dynamic/templates.asciidoc",
    "mapping/explicit-mapping.asciidoc",
    "mapping/runtime.asciidoc",
    "mapping/runtime.asciidoc",
    "mapping/types.asciidoc",
    "mapping/types/aggregate-metric-double.asciidoc",
    "mapping/types/alias.asciidoc",
    "mapping/types/array.asciidoc",
    "mapping/types/binary.asciidoc",
    "mapping/types/boolean.asciidoc",
    "mapping/types/completion.asciidoc",
    "mapping/types/date_nanos.asciidoc",
    "mapping/types/dense-vector.asciidoc",
    "mapping/types/flattened.asciidoc",
    "mapping/types/geo-point.asciidoc",
    "mapping/types/geo-shape.asciidoc",
    "mapping/types/histogram.asciidoc",
    "mapping/types/ip.asciidoc",
    "mapping/types/parent-join.asciidoc",
    "mapping/types/object.asciidoc",
    "mapping/types/percolator.asciidoc",
    "mapping/types/point.asciidoc",
    "mapping/types/range.asciidoc",
    "mapping/types/rank-feature.asciidoc",
    "mapping/types/rank-features.asciidoc",
    "mapping/types/search-as-you-type.asciidoc",
    "mapping/types/semantic-text.asciidoc",
    "mapping/types/shape.asciidoc",
    "mapping/types/sparse-vector.asciidoc",
    "mapping/types/text.asciidoc",
    "mapping/types/token-count.asciidoc",
    "mapping/types/unsigned_long.asciidoc",
    "mapping/types/version.asciidoc",
]


ParsedSource = collections.namedtuple("ParsedSource", ["api", "params", "body"])


def blacken(filename):
    runner = CliRunner()
    result = runner.invoke(
        black.main, [str(filename), "--line-length=75", "--target-version=py37"]
    )
    assert result.exit_code == 0, result.output


def main():
    for filepath in asciidocs_dir.iterdir():
        if filepath.name.endswith(".asciidoc"):
            filepath.unlink()

    if not flight_recorder_dir.exists() or not report_path.exists():
        raise RuntimeError(
            f"clients-flight-recorder repository not checked out at {flight_recorder_dir}"
        )

    with report_path.open() as f:
        report = json.loads(f.read())

    t = jinja_env.get_template("example")

    for exm in report:
        if exm["lang"] != "console":
            continue
        if exm["source_location"]["file"] not in files_to_generate:
            continue

        parsed_sources = []
        for src in exm["parsed_source"]:
            params = (src.get("params") or {}).copy()
            params.update(src.get("query") or {})
            params = {
                k: (list(v.split(",")) if isinstance(v, str) and "," in v else v)
                for k, v in params.items()
            }

            parsed_sources.append(
                ParsedSource(
                    api=src["api"],
                    params={
                        substitutions.get(k, k): repr(v) for k, v in params.items()
                    },
                    body=src.get("body", None) or None,
                )
            )

        with tempfile.NamedTemporaryFile("w+", delete=False) as tmp_file:
            tmp_file.write(t.render(parsed_sources=parsed_sources))

        try:
            blacken(tmp_file.name)
        except AssertionError:
            loc = exm["source_location"]
            print(f"Failed to format {loc['file']}:{loc['line']}, skipping.")
            continue

        with open(tmp_file.name) as f:
            data = f.read()
            data = data.rstrip().replace(",)", ")")

        os.unlink(tmp_file.name)

        with (asciidocs_dir / f"{exm['digest']}.asciidoc").open(mode="w") as f:
            f.truncate()
            f.write(
                f"""// {exm['source_location']['file']}:{exm['source_location']['line']}

[source, python]
----
{data}
----"""
            )


if __name__ == "__main__":
    main()