providers/elasticsearch/provider.yaml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

---
package-name: apache-airflow-providers-elasticsearch
name: Elasticsearch
description: |
    `Elasticsearch <https://www.elastic.co/elasticsearch>`__

state: ready
source-date-epoch: 1744791912
# Note that those versions are maintained by release manager - do not update them manually
# with the exception of case where other provider in sources has >= new provider version.
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
# to be done in the same PR
versions:
  - 6.2.2
  - 6.2.1
  - 6.2.0
  - 6.0.0
  - 5.5.3
  - 5.5.2
  - 5.5.1
  - 5.5.0
  - 5.4.2
  - 5.4.1
  - 5.4.0
  - 5.3.4
  - 5.3.3
  - 5.3.2
  - 5.3.1
  - 5.3.0
  - 5.2.0
  - 5.1.1
  - 5.1.0
  - 5.0.2
  - 5.0.1
  - 5.0.0
  - 4.5.1
  - 4.5.0
  - 4.4.0
  - 4.3.3
  - 4.3.2
  - 4.3.1
  - 4.3.0
  - 4.2.1
  - 4.2.0
  - 4.1.0
  - 4.0.0
  - 3.0.3
  - 3.0.2
  - 3.0.1
  - 3.0.0
  - 2.2.0
  - 2.1.0
  - 2.0.3
  - 2.0.2
  - 2.0.1
  - 1.0.4
  - 1.0.3
  - 1.0.2
  - 1.0.1
  - 1.0.0

integrations:
  - integration-name: Elasticsearch
    external-doc-url: https://www.elastic.co/elasticsearch
    logo: /docs/integration-logos/Elasticsearch.png
    tags: [software]

hooks:
  - integration-name: Elasticsearch
    python-modules:
      - airflow.providers.elasticsearch.hooks.elasticsearch

connection-types:
  - hook-class-name: airflow.providers.elasticsearch.hooks.elasticsearch.ElasticsearchSQLHook
    connection-type: elasticsearch

logging:
  - airflow.providers.elasticsearch.log.es_task_handler.ElasticsearchTaskHandler

config:
  elasticsearch:
    description: ~
    options:
      host:
        description: |
          Elasticsearch host
        version_added: 1.10.4
        type: string
        example: ~
        default: ""
      log_id_template:
        description: |
          Format of the log_id, which is used to query for a given tasks logs
        version_added: 1.10.4
        type: string
        example: ~
        is_template: true
        default: "{dag_id}-{task_id}-{run_id}-{map_index}-{try_number}"
      end_of_log_mark:
        description: |
          Used to mark the end of a log stream for a task
        version_added: 1.10.4
        type: string
        example: ~
        default: "end_of_log"
      frontend:
        description: |
          Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id
          Code will construct log_id using the log_id template from the argument above.
          NOTE: scheme will default to https if one is not provided
        version_added: 1.10.4
        type: string
        example: "http://localhost:5601/app/kibana#/discover\
          ?_a=(columns:!(message),query:(language:kuery,query:'log_id: \"{log_id}\"'),sort:!(log.offset,asc))"
        default: ""
      write_stdout:
        description: |
          Write the task logs to the stdout of the worker, rather than the default files
        version_added: 1.10.4
        type: string
        example: ~
        default: "False"
      write_to_es:
        description: |
          Write the task logs to the ElasticSearch
        version_added: 5.5.4
        type: string
        example: ~
        default: "False"
      target_index:
        description: |
          Name of the index to write to, when enabling writing the task logs to the ElasticSearch
        version_added: 5.5.4
        type: string
        example: ~
        default: "airflow-logs"
      json_format:
        description: |
          Instead of the default log formatter, write the log lines as JSON
        version_added: 1.10.4
        type: string
        example: ~
        default: "False"
      json_fields:
        description: |
          Log fields to also attach to the json output, if enabled
        version_added: 1.10.4
        type: string
        example: ~
        default: "asctime, filename, lineno, levelname, message"
      host_field:
        description: |
          The field where host name is stored (normally either `host` or `host.name`)
        version_added: 2.1.1
        type: string
        example: ~
        default: "host"
      offset_field:
        description: |
          The field where offset is stored (normally either `offset` or `log.offset`)
        version_added: 2.1.1
        type: string
        example: ~
        default: "offset"
      index_patterns:
        description: |
          Comma separated list of index patterns to use when searching for logs (default: `_all`).
          The index_patterns_callable takes precedence over this.
        version_added: 2.6.0
        type: string
        example: something-*
        default: "_all"
      index_patterns_callable:
        description: |
          A string representing the full path to the Python callable path which accept TI object and
          return comma separated list of index patterns. This will takes precedence over index_patterns.
        version_added: 5.5.0
        type: string
        example: module.callable
        default: ""
  elasticsearch_configs:
    description: ~
    options:
      http_compress:
        description: ~
        version_added: 1.10.5
        type: string
        example: ~
        default: "False"
      verify_certs:
        description: ~
        version_added: 1.10.5
        type: string
        example: ~
        default: "True"