images/airflow/2.9.2/docker-compose-hybrid.yaml

version: "3.8"

x-airflow-common: &airflow-common
  image: amazon-mwaa-docker-images/airflow:2.9.2-dev
  restart: always
  init: true
  environment:
    # AWS credentials
    AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
    AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
    AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
    AWS_REGION: "us-west-2"
    AWS_DEFAULT_REGION: "us-west-2"

    # > MWAA Configurations
    # >> Core Configuration
    # Use this environment variable if you have custom requirements you want to install when the
    # container is up.
    MWAA__CORE__REQUIREMENTS_PATH: ${MWAA__CORE__REQUIREMENTS_PATH}
    MWAA__CORE__AUTH_TYPE: "testing"
    # Additional Airflow configuration can be passed here in JSON form.
    MWAA__CORE__CREATED_AT: "Tue Sep 18 23:05:58 UTC 2024"
    MWAA__CORE__CUSTOM_AIRFLOW_CONFIGS: "{}"
    MWAA__CORE__FERNET_KEY: '{"FernetKey": "fake-key-nNge+lks3RBeGVrnZ1Dq5GjKerbZKmb7dXNnsNsGy3E="}'
    MWAA__WEBSERVER__SECRET: '{"secret_key": "fake-key-aYDdF6d+Fjznai5yBW63CUAi0IipJqDHlNSWIun6y8o="}'
    # Use this enviornment variable to enable encryption with KMS.
    MWAA__CORE__KMS_KEY_ARN: ${MWAA__CORE__KMS_KEY_ARN}
    # >> Database configuration
    MWAA__DB__CREDENTIALS: '{"username": "airflow", "password": "airflow"}'
    MWAA__DB__POSTGRES_DB: "airflow"
    MWAA__DB__POSTGRES_HOST: "postgres"
    MWAA__DB__POSTGRES_PORT: "5432"
    MWAA__DB__POSTGRES_SSLMODE: "prefer"
    # >> SQS configuration
    MWAA__SQS__CREATE_QUEUE: True
    MWAA__SQS__CUSTOM_ENDPOINT: http://sqs:9324
    MWAA__SQS__QUEUE_URL: http://sqs:9324/000000000000/celery-queue
    MWAA__SQS__USE_SSL: False
    # >> Logging configuration.
    # Use these environment variables if you want to publish logs to CloudWatch.
    MWAA__LOGGING__AIRFLOW_DAGPROCESSOR_LOGS_ENABLED: ${MWAA__LOGGING__AIRFLOW_DAGPROCESSOR_LOGS_ENABLED}
    MWAA__LOGGING__AIRFLOW_DAGPROCESSOR_LOG_GROUP_ARN: ${MWAA__LOGGING__AIRFLOW_DAGPROCESSOR_LOG_GROUP_ARN}
    MWAA__LOGGING__AIRFLOW_DAGPROCESSOR_LOG_LEVEL: ${MWAA__LOGGING__AIRFLOW_DAGPROCESSOR_LOG_LEVEL}
    MWAA__LOGGING__AIRFLOW_SCHEDULER_LOGS_ENABLED: ${MWAA__LOGGING__AIRFLOW_SCHEDULER_LOGS_ENABLED}
    MWAA__LOGGING__AIRFLOW_SCHEDULER_LOG_GROUP_ARN: ${MWAA__LOGGING__AIRFLOW_SCHEDULER_LOG_GROUP_ARN}
    MWAA__LOGGING__AIRFLOW_SCHEDULER_LOG_LEVEL: ${MWAA__LOGGING__AIRFLOW_SCHEDULER_LOG_LEVEL}
    MWAA__LOGGING__AIRFLOW_TRIGGERER_LOGS_ENABLED: ${MWAA__LOGGING__AIRFLOW_TRIGGERER_LOGS_ENABLED}
    MWAA__LOGGING__AIRFLOW_TRIGGERER_LOG_GROUP_ARN: ${MWAA__LOGGING__AIRFLOW_TRIGGERER_LOG_GROUP_ARN}
    MWAA__LOGGING__AIRFLOW_TRIGGERER_LOG_LEVEL: ${MWAA__LOGGING__AIRFLOW_TRIGGERER_LOG_LEVEL}
    MWAA__LOGGING__AIRFLOW_TASK_LOGS_ENABLED: ${MWAA__LOGGING__AIRFLOW_TASK_LOGS_ENABLED}
    MWAA__LOGGING__AIRFLOW_TASK_LOG_GROUP_ARN: ${MWAA__LOGGING__AIRFLOW_TASK_LOG_GROUP_ARN}
    MWAA__LOGGING__AIRFLOW_TASK_LOG_LEVEL: ${MWAA__LOGGING__AIRFLOW_TASK_LOG_LEVEL}
    MWAA__LOGGING__AIRFLOW_WEBSERVER_LOGS_ENABLED: ${MWAA__LOGGING__AIRFLOW_WEBSERVER_LOGS_ENABLED}
    MWAA__LOGGING__AIRFLOW_WEBSERVER_LOG_GROUP_ARN: ${MWAA__LOGGING__AIRFLOW_WEBSERVER_LOG_GROUP_ARN}
    MWAA__LOGGING__AIRFLOW_WEBSERVER_LOG_LEVEL: ${MWAA__LOGGING__AIRFLOW_WEBSERVER_LOG_LEVEL}
    MWAA__LOGGING__AIRFLOW_WORKER_LOGS_ENABLED: ${MWAA__LOGGING__AIRFLOW_WORKER_LOGS_ENABLED}
    MWAA__LOGGING__AIRFLOW_WORKER_LOG_GROUP_ARN: ${MWAA__LOGGING__AIRFLOW_WORKER_LOG_GROUP_ARN}
    MWAA__LOGGING__AIRFLOW_WORKER_LOG_LEVEL: ${MWAA__LOGGING__AIRFLOW_WORKER_LOG_LEVEL}

  volumes:
    - ./dags:/usr/local/airflow/dags
    - ./plugins:/usr/local/airflow/plugins
    - ./requirements:/usr/local/airflow/requirements
    - ./startup:/usr/local/airflow/startup

  depends_on: &airflow-common-depends-on
    postgres:
      condition: service_healthy
    sqs:
      condition: service_healthy

services:
  postgres:
    image: postgres:13
    container_name: mwaa-292-db
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    volumes:
      - postgres-db-volume:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "airflow"]
      interval: 10s
      retries: 5
      start_period: 5s
    restart: always
    ports:
      - 5432:5432
    expose:
      - 5432

  sqs:
    image: softwaremill/elasticmq:latest
    healthcheck:
      # https://github.com/softwaremill/elasticmq/issues/776#issuecomment-1582527921
      test: ["CMD-SHELL", "wget -q -S -O - 127.0.0.1:9324/?Action=ListQueues"]
      interval: 10s
      retries: 5
      start_period: 5s
    ports:
      - 9324:9324
      - 9325:9325
    expose:
      - 9324
      - 9325

  spy:
    <<: *airflow-common
    command: spy
    container_name: mwaa-292-spy

  webserver:
    <<: *airflow-common
    command: webserver
    container_name: mwaa-292-webserver
    ports:
      - 8080:8080
    expose:
      - 8080

  # Launches a scheduler/worker hybrid container
  hybrid:
    <<: *airflow-common
    command: hybrid
    container_name: mwaa-292-hybrid

volumes:
  postgres-db-volume:
    name: "mwaa-292-db-volume"