providers/amazon/provider.yaml (1,126 lines of code) (raw):

# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. --- package-name: apache-airflow-providers-amazon name: Amazon description: | Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__). state: ready source-date-epoch: 1744788746 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: - 9.7.0 - 9.6.1 - 9.6.0 - 9.5.0 - 9.4.0 - 9.2.0 - 9.1.0 - 9.0.0 - 8.29.0 - 8.28.0 - 8.27.0 - 8.26.0 - 8.25.0 - 8.24.0 - 8.23.0 - 8.22.0 - 8.21.0 - 8.20.0 - 8.19.0 - 8.18.0 - 8.17.0 - 8.16.0 - 8.15.0 - 8.14.0 - 8.13.0 - 8.12.0 - 8.11.0 - 8.10.0 - 8.9.0 - 8.8.0 - 8.7.1 - 8.7.0 - 8.6.0 - 8.5.1 - 8.5.0 - 8.4.0 - 8.3.1 - 8.3.0 - 8.2.0 - 8.1.0 - 8.0.0 - 7.4.1 - 7.4.0 - 7.3.0 - 7.2.1 - 7.2.0 - 7.1.0 - 7.0.0 - 6.2.0 - 6.1.0 - 6.0.0 - 5.1.0 - 5.0.0 - 4.1.0 - 4.0.0 - 3.4.0 - 3.3.0 - 3.2.0 - 3.1.1 - 3.0.0 - 2.6.0 - 2.5.0 - 2.4.0 - 2.3.0 - 2.2.0 - 2.1.0 - 2.0.0 - 1.4.0 - 1.3.0 - 1.2.0 - 1.1.0 - 1.0.0 integrations: - integration-name: Amazon Athena external-doc-url: https://aws.amazon.com/athena/ logo: /docs/integration-logos/Amazon-Athena_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/athena/athena_boto.rst - /docs/apache-airflow-providers-amazon/operators/athena/athena_sql.rst tags: [aws] - integration-name: Amazon Bedrock external-doc-url: https://aws.amazon.com/bedrock/ logo: /docs/integration-logos/Amazon-Bedrock_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/bedrock.rst tags: [aws] - integration-name: Amazon Chime external-doc-url: https://aws.amazon.com/chime/ logo: /docs/integration-logos/Amazon-Chime-light-bg.png tags: [aws] - integration-name: Amazon CloudFormation external-doc-url: https://aws.amazon.com/cloudformation/ logo: /docs/integration-logos/AWS-CloudFormation_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/cloudformation.rst tags: [aws] - integration-name: Amazon CloudWatch Logs external-doc-url: https://aws.amazon.com/cloudwatch/ logo: /docs/integration-logos/Amazon-CloudWatch_light-bg@4x.png tags: [aws] - integration-name: Amazon Comprehend external-doc-url: https://aws.amazon.com/comprehend/ logo: /docs/integration-logos/Amazon-Comprehend_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/comprehend.rst tags: [aws] - integration-name: Amazon DataSync external-doc-url: https://aws.amazon.com/datasync/ how-to-guide: - /docs/apache-airflow-providers-amazon/operators/datasync.rst tags: [aws] - integration-name: Amazon DynamoDB external-doc-url: https://aws.amazon.com/dynamodb/ logo: /docs/integration-logos/Amazon-DynamoDB_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/dynamodb.rst tags: [aws] - integration-name: Amazon EC2 external-doc-url: https://aws.amazon.com/ec2/ logo: /docs/integration-logos/Amazon-EC2_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/ec2.rst tags: [aws] - integration-name: Amazon Elastic Container Registry (ECR) external-doc-url: https://aws.amazon.com/ecr/ logo: /docs/integration-logos/Amazon-Elastic-Container-Registry_light-bg@4x.png tags: [aws] - integration-name: Amazon ECS external-doc-url: https://aws.amazon.com/ecs/ logo: /docs/integration-logos/Amazon-Elastic-Container-Service_light-bg@4x.png tags: [aws] - integration-name: Amazon Elastic Kubernetes Service (EKS) external-doc-url: https://aws.amazon.com/eks/ logo: /docs/integration-logos/Amazon-Elastic-Kubernetes-Service_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/eks.rst tags: [aws] - integration-name: Amazon ElastiCache external-doc-url: https://aws.amazon.com/elasticache/redis// logo: /docs/integration-logos/Amazon-ElastiCache_light-bg@4x.png tags: [aws] - integration-name: Amazon EMR external-doc-url: https://aws.amazon.com/emr/ how-to-guide: - /docs/apache-airflow-providers-amazon/operators/emr/emr.rst logo: /docs/integration-logos/Amazon-EMR_light-bg@4x.png tags: [aws] - integration-name: Amazon EMR on EKS external-doc-url: https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/emr-eks.html how-to-guide: - /docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst logo: /docs/integration-logos/Amazon-EMR_light-bg@4x.png tags: [aws] - integration-name: Amazon EMR Serverless external-doc-url: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/emr-serverless.html how-to-guide: - /docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst logo: /docs/integration-logos/Amazon-EMR_light-bg@4x.png tags: [aws] - integration-name: Amazon EventBridge external-doc-url: https://docs.aws.amazon.com/eventbridge/latest/APIReference/Welcome.html how-to-guide: - /docs/apache-airflow-providers-amazon/operators/eventbridge.rst logo: /docs/integration-logos/Amazon-EventBridge_64.png tags: [aws] - integration-name: Amazon Glacier external-doc-url: https://aws.amazon.com/glacier/ logo: /docs/integration-logos/Amazon-S3-Glacier_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/s3/glacier.rst tags: [aws] - integration-name: Amazon Kinesis Data Firehose external-doc-url: https://aws.amazon.com/kinesis/data-firehose/ logo: /docs/integration-logos/Amazon-Kinesis-Data-Firehose_light-bg@4x.png tags: [aws] - integration-name: Amazon Managed Service for Apache Flink how-to-guide: - /docs/apache-airflow-providers-amazon/operators/kinesis_analytics.rst external-doc-url: https://aws.amazon.com/managed-service-apache-flink/ logo: /docs/integration-logos/Amazon-Kinesis-Analytics_light-bg@4x.png tags: [aws] - integration-name: Amazon OpenSearch Serverless how-to-guide: - /docs/apache-airflow-providers-amazon/operators/opensearchserverless.rst external-doc-url: https://aws.amazon.com/opensearchserverless/ logo: /docs/integration-logos/Amazon-OpenSearch_light-bg@4x.png tags: [aws] - integration-name: Amazon RDS external-doc-url: https://aws.amazon.com/rds/ logo: /docs/integration-logos/Amazon-RDS_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/rds.rst tags: [aws] - integration-name: Amazon Redshift external-doc-url: https://aws.amazon.com/redshift/ logo: /docs/integration-logos/Amazon-Redshift_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst - /docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst tags: [aws] - integration-name: Amazon Redshift Data external-doc-url: https://aws.amazon.com/redshift/ logo: /docs/integration-logos/Amazon-Redshift_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst tags: [aws] - integration-name: Amazon SageMaker external-doc-url: https://aws.amazon.com/sagemaker/ logo: /docs/integration-logos/Amazon-SageMaker_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/sagemaker.rst tags: [aws] - integration-name: Amazon SageMaker Unified Studio external-doc-url: https://aws.amazon.com/sagemaker/unified-studio/ logo: /docs/integration-logos/Amazon-SageMaker_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/sagemakerunifiedstudio.rst tags: [aws] - integration-name: Amazon SecretsManager external-doc-url: https://aws.amazon.com/secrets-manager/ logo: /docs/integration-logos/AWS-Secrets-Manager_light-bg@4x.png tags: [aws] - integration-name: Amazon Simple Email Service (SES) external-doc-url: https://aws.amazon.com/ses/ logo: /docs/integration-logos/Amazon-Simple-Email-Service-SES_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/ecs.rst tags: [aws] - integration-name: Amazon Simple Notification Service (SNS) external-doc-url: https://aws.amazon.com/sns/ logo: /docs/integration-logos/Amazon-Simple-Notification-Service-SNS_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/sns.rst tags: [aws] - integration-name: Amazon Simple Queue Service (SQS) external-doc-url: https://aws.amazon.com/sqs/ logo: /docs/integration-logos/Amazon-Simple-Queue-Service-SQS_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/sqs.rst tags: [aws] - integration-name: Amazon Simple Storage Service (S3) external-doc-url: https://aws.amazon.com/s3/ logo: /docs/integration-logos/Amazon-Simple-Storage-Service-S3_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/s3/s3.rst tags: [aws] - integration-name: Amazon Systems Manager (SSM) external-doc-url: https://aws.amazon.com/systems-manager/ logo: /docs/integration-logos/AWS-Systems-Manager_light-bg@4x.png tags: [aws] - integration-name: Amazon Web Services external-doc-url: https://aws.amazon.com/ logo: /docs/integration-logos/AWS-Cloud-alt_light-bg@4x.png tags: [aws] - integration-name: AWS Batch external-doc-url: https://aws.amazon.com/batch/ logo: /docs/integration-logos/AWS-Batch_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/batch.rst tags: [aws] - integration-name: AWS DataSync external-doc-url: https://aws.amazon.com/datasync/ logo: /docs/integration-logos/AWS-DataSync_light-bg@4x.png tags: [aws] - integration-name: AWS Glue external-doc-url: https://aws.amazon.com/glue/ logo: /docs/integration-logos/AWS-Glue_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/glue.rst tags: [aws] - integration-name: AWS Lambda external-doc-url: https://aws.amazon.com/lambda/ logo: /docs/integration-logos/AWS-Lambda_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/lambda.rst tags: [aws] - integration-name: AWS Step Functions external-doc-url: https://aws.amazon.com/step-functions/ logo: /docs/integration-logos/AWS-Step-Functions_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/step_functions.rst tags: [aws] - integration-name: AWS Database Migration Service external-doc-url: https://aws.amazon.com/dms/ logo: /docs/integration-logos/AWS-Database-Migration-Service_64@5x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/dms.rst tags: [aws] - integration-name: Amazon QuickSight external-doc-url: https://aws.amazon.com/quicksight/ logo: /docs/integration-logos/Amazon-Quicksight_light-bg@4x.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/quicksight.rst tags: [aws] - integration-name: AWS Security Token Service (STS) external-doc-url: https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html logo: /docs/integration-logos/AWS-STS_light-bg@4x.png tags: [aws] - integration-name: Amazon Appflow external-doc-url: https://docs.aws.amazon.com/appflow/1.0/APIReference/Welcome.html logo: /docs/integration-logos/Amazon_AppFlow_light.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/appflow.rst tags: [aws] - integration-name: AWS Glue DataBrew external-doc-url: https://docs.aws.amazon.com/databrew/latest/dg/what-is.html how-to-guide: - /docs/apache-airflow-providers-amazon/operators/glue_databrew.rst logo: /docs/integration-logos/AWS-Glue-DataBrew_64.png tags: [aws] - integration-name: Amazon Verified Permissions external-doc-url: https://aws.amazon.com/verified-permissions/ logo: /docs/integration-logos/Amazon-Verified-Permissions.png tags: [aws] - integration-name: Amazon Neptune external-doc-url: https://aws.amazon.com/neptune/ logo: /docs/integration-logos/Amazon-Neptune_64.png how-to-guide: - /docs/apache-airflow-providers-amazon/operators/neptune.rst tags: [aws] operators: - integration-name: Amazon Athena python-modules: - airflow.providers.amazon.aws.operators.athena - integration-name: Amazon Web Services python-modules: - airflow.providers.amazon.aws.operators.base_aws - integration-name: AWS Batch python-modules: - airflow.providers.amazon.aws.operators.batch - integration-name: Amazon Bedrock python-modules: - airflow.providers.amazon.aws.operators.bedrock - integration-name: Amazon CloudFormation python-modules: - airflow.providers.amazon.aws.operators.cloud_formation - integration-name: Amazon Comprehend python-modules: - airflow.providers.amazon.aws.operators.comprehend - integration-name: Amazon DataSync python-modules: - airflow.providers.amazon.aws.operators.datasync - integration-name: AWS Database Migration Service python-modules: - airflow.providers.amazon.aws.operators.dms - integration-name: Amazon EC2 python-modules: - airflow.providers.amazon.aws.operators.ec2 - integration-name: Amazon ECS python-modules: - airflow.providers.amazon.aws.operators.ecs - integration-name: Amazon Elastic Kubernetes Service (EKS) python-modules: - airflow.providers.amazon.aws.operators.eks - integration-name: Amazon EMR python-modules: - airflow.providers.amazon.aws.operators.emr - integration-name: Amazon EMR on EKS python-modules: - airflow.providers.amazon.aws.operators.emr - integration-name: Amazon EventBridge python-modules: - airflow.providers.amazon.aws.operators.eventbridge - integration-name: Amazon Glacier python-modules: - airflow.providers.amazon.aws.operators.glacier - integration-name: AWS Glue python-modules: - airflow.providers.amazon.aws.operators.glue - airflow.providers.amazon.aws.operators.glue_crawler - integration-name: AWS Lambda python-modules: - airflow.providers.amazon.aws.operators.lambda_function - integration-name: Amazon Managed Service for Apache Flink python-modules: - airflow.providers.amazon.aws.operators.kinesis_analytics - integration-name: Amazon Managed Workflows for Apache Airflow (MWAA) python-modules: - airflow.providers.amazon.aws.operators.mwaa - integration-name: Amazon Simple Storage Service (S3) python-modules: - airflow.providers.amazon.aws.operators.s3 - integration-name: Amazon SageMaker python-modules: - airflow.providers.amazon.aws.operators.sagemaker - integration-name: Amazon SageMaker Unified Studio python-modules: - airflow.providers.amazon.aws.operators.sagemaker_unified_studio - integration-name: Amazon Simple Notification Service (SNS) python-modules: - airflow.providers.amazon.aws.operators.sns - integration-name: Amazon Simple Queue Service (SQS) python-modules: - airflow.providers.amazon.aws.operators.sqs - integration-name: AWS Step Functions python-modules: - airflow.providers.amazon.aws.operators.step_function - integration-name: Amazon RDS python-modules: - airflow.providers.amazon.aws.operators.rds - integration-name: Amazon Redshift python-modules: - airflow.providers.amazon.aws.operators.redshift_cluster - airflow.providers.amazon.aws.operators.redshift_data - integration-name: Amazon QuickSight python-modules: - airflow.providers.amazon.aws.operators.quicksight - integration-name: Amazon Appflow python-modules: - airflow.providers.amazon.aws.operators.appflow - integration-name: AWS Glue DataBrew python-modules: - airflow.providers.amazon.aws.operators.glue_databrew - integration-name: Amazon Neptune python-modules: - airflow.providers.amazon.aws.operators.neptune sensors: - integration-name: Amazon Athena python-modules: - airflow.providers.amazon.aws.sensors.athena - integration-name: Amazon Web Services python-modules: - airflow.providers.amazon.aws.sensors.base_aws - integration-name: AWS Batch python-modules: - airflow.providers.amazon.aws.sensors.batch - integration-name: Amazon Bedrock python-modules: - airflow.providers.amazon.aws.sensors.bedrock - integration-name: Amazon CloudFormation python-modules: - airflow.providers.amazon.aws.sensors.cloud_formation - integration-name: Amazon Comprehend python-modules: - airflow.providers.amazon.aws.sensors.comprehend - integration-name: AWS Database Migration Service python-modules: - airflow.providers.amazon.aws.sensors.dms - integration-name: Amazon DynamoDB python-modules: - airflow.providers.amazon.aws.sensors.dynamodb - integration-name: Amazon EC2 python-modules: - airflow.providers.amazon.aws.sensors.ec2 - integration-name: Amazon ECS python-modules: - airflow.providers.amazon.aws.sensors.ecs - integration-name: Amazon Elastic Kubernetes Service (EKS) python-modules: - airflow.providers.amazon.aws.sensors.eks - integration-name: Amazon EMR python-modules: - airflow.providers.amazon.aws.sensors.emr - integration-name: Amazon EMR on EKS python-modules: - airflow.providers.amazon.aws.sensors.emr - integration-name: Amazon Glacier python-modules: - airflow.providers.amazon.aws.sensors.glacier - integration-name: AWS Glue python-modules: - airflow.providers.amazon.aws.sensors.glue - airflow.providers.amazon.aws.sensors.glue_crawler - airflow.providers.amazon.aws.sensors.glue_catalog_partition - integration-name: AWS Lambda python-modules: - airflow.providers.amazon.aws.sensors.lambda_function - integration-name: Amazon Managed Service for Apache Flink python-modules: - airflow.providers.amazon.aws.sensors.kinesis_analytics - integration-name: Amazon Managed Workflows for Apache Airflow (MWAA) python-modules: - airflow.providers.amazon.aws.sensors.mwaa - integration-name: Amazon OpenSearch Serverless python-modules: - airflow.providers.amazon.aws.sensors.opensearch_serverless - integration-name: Amazon RDS python-modules: - airflow.providers.amazon.aws.sensors.rds - integration-name: Amazon Redshift python-modules: - airflow.providers.amazon.aws.sensors.redshift_cluster - integration-name: Amazon Simple Storage Service (S3) python-modules: - airflow.providers.amazon.aws.sensors.s3 - integration-name: Amazon SageMaker python-modules: - airflow.providers.amazon.aws.sensors.sagemaker - integration-name: Amazon SageMaker Unified Studio python-modules: - airflow.providers.amazon.aws.sensors.sagemaker_unified_studio - integration-name: Amazon Simple Queue Service (SQS) python-modules: - airflow.providers.amazon.aws.sensors.sqs - integration-name: AWS Step Functions python-modules: - airflow.providers.amazon.aws.sensors.step_function - integration-name: Amazon QuickSight python-modules: - airflow.providers.amazon.aws.sensors.quicksight asset-uris: - schemes: [s3] handler: airflow.providers.amazon.aws.assets.s3.sanitize_uri to_openlineage_converter: airflow.providers.amazon.aws.assets.s3.convert_asset_to_openlineage factory: airflow.providers.amazon.aws.assets.s3.create_asset # dataset has been renamed to asset in Airflow 3.0 # This is kept for backward compatibility. dataset-uris: - schemes: [s3] handler: airflow.providers.amazon.aws.assets.s3.sanitize_uri to_openlineage_converter: airflow.providers.amazon.aws.assets.s3.convert_asset_to_openlineage factory: airflow.providers.amazon.aws.assets.s3.create_asset filesystems: - airflow.providers.amazon.aws.fs.s3 hooks: - integration-name: Amazon Athena python-modules: - airflow.providers.amazon.aws.hooks.athena - airflow.providers.amazon.aws.hooks.athena_sql - integration-name: Amazon Bedrock python-modules: - airflow.providers.amazon.aws.hooks.bedrock - integration-name: Amazon Chime python-modules: - airflow.providers.amazon.aws.hooks.chime - integration-name: Amazon Comprehend python-modules: - airflow.providers.amazon.aws.hooks.comprehend - integration-name: Amazon DynamoDB python-modules: - airflow.providers.amazon.aws.hooks.dynamodb - integration-name: Amazon Web Services python-modules: - airflow.providers.amazon.aws.hooks.base_aws - integration-name: AWS Batch python-modules: - airflow.providers.amazon.aws.hooks.batch_client - airflow.providers.amazon.aws.hooks.batch_waiters - integration-name: Amazon CloudFormation python-modules: - airflow.providers.amazon.aws.hooks.cloud_formation - integration-name: Amazon DataSync python-modules: - airflow.providers.amazon.aws.hooks.datasync - integration-name: AWS Database Migration Service python-modules: - airflow.providers.amazon.aws.hooks.dms - integration-name: Amazon EC2 python-modules: - airflow.providers.amazon.aws.hooks.ec2 - integration-name: Amazon Elastic Container Registry (ECR) python-modules: - airflow.providers.amazon.aws.hooks.ecr - integration-name: Amazon ECS python-modules: - airflow.providers.amazon.aws.hooks.ecs - integration-name: Amazon ElastiCache python-modules: - airflow.providers.amazon.aws.hooks.elasticache_replication_group - integration-name: Amazon Elastic Kubernetes Service (EKS) python-modules: - airflow.providers.amazon.aws.hooks.eks - integration-name: Amazon EMR python-modules: - airflow.providers.amazon.aws.hooks.emr - integration-name: Amazon EMR on EKS python-modules: - airflow.providers.amazon.aws.hooks.emr - integration-name: Amazon EventBridge python-modules: - airflow.providers.amazon.aws.hooks.eventbridge - integration-name: Amazon Glacier python-modules: - airflow.providers.amazon.aws.hooks.glacier - integration-name: AWS Glue python-modules: - airflow.providers.amazon.aws.hooks.glue - airflow.providers.amazon.aws.hooks.glue_crawler - airflow.providers.amazon.aws.hooks.glue_catalog - integration-name: Amazon Kinesis Data Firehose python-modules: - airflow.providers.amazon.aws.hooks.kinesis - integration-name: AWS Lambda python-modules: - airflow.providers.amazon.aws.hooks.lambda_function - integration-name: Amazon Managed Service for Apache Flink python-modules: - airflow.providers.amazon.aws.hooks.kinesis_analytics - integration-name: Amazon CloudWatch Logs python-modules: - airflow.providers.amazon.aws.hooks.logs - integration-name: Amazon Managed Workflows for Apache Airflow (MWAA) python-modules: - airflow.providers.amazon.aws.hooks.mwaa - integration-name: Amazon OpenSearch Serverless python-modules: - airflow.providers.amazon.aws.hooks.opensearch_serverless - integration-name: Amazon RDS python-modules: - airflow.providers.amazon.aws.hooks.rds - integration-name: Amazon Redshift python-modules: - airflow.providers.amazon.aws.hooks.redshift_sql - airflow.providers.amazon.aws.hooks.redshift_cluster - airflow.providers.amazon.aws.hooks.redshift_data - integration-name: Amazon Simple Storage Service (S3) python-modules: - airflow.providers.amazon.aws.hooks.s3 - integration-name: Amazon SageMaker python-modules: - airflow.providers.amazon.aws.hooks.sagemaker - integration-name: Amazon SageMaker Unified Studio python-modules: - airflow.providers.amazon.aws.hooks.sagemaker_unified_studio - integration-name: Amazon Simple Email Service (SES) python-modules: - airflow.providers.amazon.aws.hooks.ses - integration-name: Amazon Systems Manager (SSM) python-modules: - airflow.providers.amazon.aws.hooks.ssm - integration-name: Amazon SecretsManager python-modules: - airflow.providers.amazon.aws.hooks.secrets_manager - integration-name: Amazon Simple Notification Service (SNS) python-modules: - airflow.providers.amazon.aws.hooks.sns - integration-name: Amazon Simple Queue Service (SQS) python-modules: - airflow.providers.amazon.aws.hooks.sqs - integration-name: AWS Step Functions python-modules: - airflow.providers.amazon.aws.hooks.step_function - integration-name: Amazon QuickSight python-modules: - airflow.providers.amazon.aws.hooks.quicksight - integration-name: AWS Security Token Service (STS) python-modules: - airflow.providers.amazon.aws.hooks.sts - integration-name: Amazon Appflow python-modules: - airflow.providers.amazon.aws.hooks.appflow - integration-name: AWS Glue DataBrew python-modules: - airflow.providers.amazon.aws.hooks.glue_databrew - integration-name: Amazon Verified Permissions python-modules: - airflow.providers.amazon.aws.hooks.verified_permissions - integration-name: Amazon Neptune python-modules: - airflow.providers.amazon.aws.hooks.neptune triggers: - integration-name: Amazon Web Services python-modules: - airflow.providers.amazon.aws.triggers.base - integration-name: Amazon Athena python-modules: - airflow.providers.amazon.aws.triggers.athena - integration-name: AWS Batch python-modules: - airflow.providers.amazon.aws.triggers.batch - integration-name: Amazon Bedrock python-modules: - airflow.providers.amazon.aws.triggers.bedrock - integration-name: Amazon Comprehend python-modules: - airflow.providers.amazon.aws.triggers.comprehend - integration-name: Amazon EC2 python-modules: - airflow.providers.amazon.aws.triggers.ec2 - integration-name: AWS Lambda python-modules: - airflow.providers.amazon.aws.triggers.lambda_function - integration-name: Amazon Managed Workflows for Apache Airflow (MWAA) python-modules: - airflow.providers.amazon.aws.triggers.mwaa - integration-name: Amazon Managed Service for Apache Flink python-modules: - airflow.providers.amazon.aws.triggers.kinesis_analytics - integration-name: Amazon OpenSearch Serverless python-modules: - airflow.providers.amazon.aws.triggers.opensearch_serverless - integration-name: Amazon Redshift python-modules: - airflow.providers.amazon.aws.triggers.redshift_cluster - airflow.providers.amazon.aws.triggers.redshift_data - integration-name: Amazon SageMaker python-modules: - airflow.providers.amazon.aws.triggers.sagemaker - integration-name: Amazon SageMaker Unified Studio python-modules: - airflow.providers.amazon.aws.triggers.sagemaker_unified_studio - integration-name: AWS Glue python-modules: - airflow.providers.amazon.aws.triggers.glue - airflow.providers.amazon.aws.triggers.glue_crawler - integration-name: Amazon Simple Storage Service (S3) python-modules: - airflow.providers.amazon.aws.triggers.s3 - integration-name: Amazon EMR python-modules: - airflow.providers.amazon.aws.triggers.emr - integration-name: Amazon Elastic Kubernetes Service (EKS) python-modules: - airflow.providers.amazon.aws.triggers.eks - integration-name: Amazon ECS python-modules: - airflow.providers.amazon.aws.triggers.ecs - integration-name: Amazon RDS python-modules: - airflow.providers.amazon.aws.triggers.rds - integration-name: AWS Step Functions python-modules: - airflow.providers.amazon.aws.triggers.step_function - integration-name: Amazon Simple Queue Service (SQS) python-modules: - airflow.providers.amazon.aws.triggers.sqs - integration-name: AWS Glue DataBrew python-modules: - airflow.providers.amazon.aws.triggers.glue_databrew - integration-name: Amazon Neptune python-modules: - airflow.providers.amazon.aws.triggers.neptune - integration-name: AWS Database Migration Service python-modules: - airflow.providers.amazon.aws.triggers.dms transfers: - source-integration-name: Amazon DynamoDB target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.dynamodb_to_s3 - source-integration-name: Google Cloud Storage (GCS) target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.gcs_to_s3 - source-integration-name: Amazon Glacier target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst python-module: airflow.providers.amazon.aws.transfers.glacier_to_gcs - source-integration-name: Google target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.google_api_to_s3 - source-integration-name: Apache Hive target-integration-name: Amazon DynamoDB how-to-guide: /docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst python-module: airflow.providers.amazon.aws.transfers.hive_to_dynamodb - source-integration-name: Hypertext Transfer Protocol (HTTP) target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/http_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.http_to_s3 - source-integration-name: Internet Message Access Protocol (IMAP) target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.imap_attachment_to_s3 - source-integration-name: MongoDB target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.mongo_to_s3 - source-integration-name: Amazon Redshift target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.redshift_to_s3 - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: Amazon Redshift how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst python-module: airflow.providers.amazon.aws.transfers.s3_to_redshift - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: SSH File Transfer Protocol (SFTP) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst python-module: airflow.providers.amazon.aws.transfers.s3_to_sftp - source-integration-name: SSH File Transfer Protocol (SFTP) target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.sftp_to_s3 - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: File Transfer Protocol (FTP) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst python-module: airflow.providers.amazon.aws.transfers.s3_to_ftp - source-integration-name: Exasol target-integration-name: Amazon Simple Storage Service (S3) python-module: airflow.providers.amazon.aws.transfers.exasol_to_s3 - source-integration-name: File Transfer Protocol (FTP) target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.ftp_to_s3 - source-integration-name: Salesforce target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.salesforce_to_s3 - source-integration-name: Local target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.local_to_s3 - source-integration-name: Common SQL target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.sql_to_s3 - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: Common SQL how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst python-module: airflow.providers.amazon.aws.transfers.s3_to_sql - source-integration-name: Amazon Web Services target-integration-name: Amazon Web Services python-module: airflow.providers.amazon.aws.transfers.base - source-integration-name: Microsoft Azure Blob Storage target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.azure_blob_to_s3 - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: Amazon DynamoDB how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_dynamodb.rst python-module: airflow.providers.amazon.aws.transfers.s3_to_dynamodb extra-links: - airflow.providers.amazon.aws.links.athena.AthenaQueryResultsLink - airflow.providers.amazon.aws.links.batch.BatchJobDefinitionLink - airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink - airflow.providers.amazon.aws.links.batch.BatchJobQueueLink - airflow.providers.amazon.aws.links.emr.EmrClusterLink - airflow.providers.amazon.aws.links.emr.EmrLogsLink - airflow.providers.amazon.aws.links.emr.EmrServerlessCloudWatchLogsLink - airflow.providers.amazon.aws.links.emr.EmrServerlessDashboardLink - airflow.providers.amazon.aws.links.emr.EmrServerlessLogsLink - airflow.providers.amazon.aws.links.emr.EmrServerlessS3LogsLink - airflow.providers.amazon.aws.links.glue.GlueJobRunDetailsLink - airflow.providers.amazon.aws.links.logs.CloudWatchEventsLink - airflow.providers.amazon.aws.links.sagemaker.SageMakerTransformJobLink - airflow.providers.amazon.aws.links.sagemaker_unified_studio.SageMakerUnifiedStudioLink - airflow.providers.amazon.aws.links.step_function.StateMachineDetailsLink - airflow.providers.amazon.aws.links.step_function.StateMachineExecutionsDetailsLink - airflow.providers.amazon.aws.links.comprehend.ComprehendPiiEntitiesDetectionLink - airflow.providers.amazon.aws.links.comprehend.ComprehendDocumentClassifierLink - airflow.providers.amazon.aws.links.datasync.DataSyncTaskLink - airflow.providers.amazon.aws.links.datasync.DataSyncTaskExecutionLink - airflow.providers.amazon.aws.links.ec2.EC2InstanceLink - airflow.providers.amazon.aws.links.ec2.EC2InstanceDashboardLink connection-types: - hook-class-name: airflow.providers.amazon.aws.hooks.base_aws.AwsGenericHook connection-type: aws - hook-class-name: airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook connection-type: chime - hook-class-name: airflow.providers.amazon.aws.hooks.emr.EmrHook connection-type: emr - hook-class-name: airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook connection-type: redshift - hook-class-name: airflow.providers.amazon.aws.hooks.athena_sql.AthenaSQLHook connection-type: athena notifications: - airflow.providers.amazon.aws.notifications.chime.ChimeNotifier - airflow.providers.amazon.aws.notifications.sns.SnsNotifier - airflow.providers.amazon.aws.notifications.sqs.SqsNotifier secrets-backends: - airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend - airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend logging: - airflow.providers.amazon.aws.log.s3_task_handler.S3TaskHandler - airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudwatchTaskHandler config: aws: description: This section contains settings for Amazon Web Services (AWS) integration. options: session_factory: description: | Full import path to the class which implements a custom session factory for ``boto3.session.Session``. For more details please have a look at :ref:`howto/connection:aws:session-factory`. default: ~ example: my_company.aws.MyCustomSessionFactory type: string version_added: 3.1.1 cloudwatch_task_handler_json_serializer: description: | By default, when logging non-string messages, all non-json objects are logged as `null`. Except `datetime` objects which are ISO formatted. Users can optionally use a `repr` serializer or provide their own JSON serializer for any non-JSON-serializable objects in the logged message. * `airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize` uses `repr` (be aware there is the potential of logging sensitive data depending on the `repr` method of logged objects) * `airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize_legacy` uses `null`. If a custom serializer is provided, it must adhere to `Callable[[Any], str | None]`, where `None` serializes to `null` (e.g. `def my_serializer(o: Any) -> str | None`). Since this is on the logging path and it's possible there's an exception being handled, special care should be taken to fail gracefully without raising a new exception inside of your serializer. type: string version_added: 8.7.2 example: airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize default: airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize_legacy aws_batch_executor: description: | This section only applies if you are using the AwsBatchExecutor in Airflow's ``[core]`` configuration. For more information on any of these execution parameters, see the link below: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/batch.html#Batch.Client.submit_job For boto3 credential management, see https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html options: conn_id: description: | The Airflow connection (i.e. credentials) used by the Batch executor to make API calls to AWS Batch. version_added: "8.11" type: string example: "aws_default" default: "aws_default" region_name: description: | The name of the AWS Region where Amazon Batch is configured. Required. version_added: "8.11" type: string example: "us-east-1" default: ~ max_submit_job_attempts: description: | The maximum number of times the Batch Executor should attempt to run a Batch Job. version_added: "8.11" type: integer example: "3" default: "3" check_health_on_startup: description: | Whether or not to check the Batch Executor health on startup. version_added: "8.11" type: boolean example: "True" default: "True" job_name: description: | The name of the job submitted to AWS Batch. It can be up to 128 letters long. The first character must be alphanumeric, can contain uppercase and lowercase letters, numbers, hyphens (-), and underscores (_). version_added: "8.11" type: string example: "airflow-batch-executor-job" default: ~ job_queue: description: | The job queue where the job is submitted. You can specify either the name or the Amazon Resource Name (ARN) of the queue. version_added: "8.11" type: string example: "airflow-batch-executor-job-queue" default: ~ job_definition: description: | The job definition used by the job. You can specify either the name or the Amazon Resource Name (ARN) of the job definition with or without the revision. If the revision is not specified, then the latest active revision is used. version_added: "8.11" type: string example: "airflow-batch-executor-job-definition" default: ~ submit_job_kwargs: description: | Additional parameters to pass to the submit_job method of the AWS Batch client. version_added: "8.11" type: string example: '{"Tags": [{"Key": "key", "Value": "value"}]}' default: ~ aws_ecs_executor: description: | This section only applies if you are using the AwsEcsExecutor in Airflow's ``[core]`` configuration. For more information on any of these execution parameters, see the link below: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs/client/run_task.html For boto3 credential management, see https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html options: conn_id: description: | The Airflow connection (i.e. credentials) used by the ECS executor to make API calls to AWS ECS. version_added: "8.10" type: string example: "aws_default" default: "aws_default" region_name: description: | The name of the AWS Region where Amazon ECS is configured. Required. version_added: "8.10" type: string example: "us-east-1" default: ~ assign_public_ip: description: | Whether to assign a public IP address to the containers launched by the ECS executor. For more info see url to Boto3 docs above. version_added: "8.10" type: boolean example: "True" default: "False" cluster: description: | Name of the Amazon ECS Cluster. Required. version_added: "8.10" type: string example: "ecs_executor_cluster" default: ~ capacity_provider_strategy: description: | The capacity provider strategy to use for the task. If a Capacity Provider Strategy is specified, the Launch Type parameter must be omitted. If no Capacity Provider Strategy or Launch Type is specified, the Default CapacityProvider Strategy for the cluster is used, if present. When you use cluster auto scaling, you must specify Capacity Provider Strategy and not Launch Type. version_added: "8.17" type: string example: "[{'capacityProvider': 'cp1', 'weight': 5}, {'capacityProvider': 'cp2', 'weight': 1}]" default: ~ container_name: description: | Name of the container that will be used to execute Airflow tasks via the ECS executor. The container should be specified in the ECS Task Definition and will receive an airflow CLI command as an additional parameter to its entrypoint. For more info see url to Boto3 docs above. Required. version_added: "8.10" type: string example: "ecs_executor_container" default: ~ launch_type: description: | Launch type can either be 'FARGATE' OR 'EC2'. For more info see url to Boto3 docs above. If a Launch Type is specified, the Capacity Provider Strategy parameter must be omitted. If no Capacity Provider Strategy or Launch Type is specified, the Default Capacity Provider Strategy for the cluster is used, if present. If the launch type is EC2, the executor will attempt to place tasks on empty EC2 instances. If there are no EC2 instances available, no task is placed and this function will be called again in the next heart-beat. If the launch type is FARGATE, this will run the tasks on new AWS Fargate instances. version_added: "8.10" type: string example: "FARGATE" default: ~ platform_version: description: | The platform version the task uses. A platform version is only specified for tasks hosted on Fargate. If one isn't specified, the LATEST platform version is used. version_added: "8.10" type: string example: "1.4.0" default: "LATEST" security_groups: description: | The comma-separated IDs of the security groups associated with the task. If you don't specify a security group, the default security group for the VPC is used. There's a limit of 5 security groups. For more info see url to Boto3 docs above. version_added: "8.10" type: string example: "sg-XXXX,sg-YYYY" default: ~ subnets: description: | The comma-separated IDs of the subnets associated with the task or service. There's a limit of 16 subnets. For more info see url to Boto3 docs above. version_added: "8.10" type: string example: "subnet-XXXXXXXX,subnet-YYYYYYYY" default: ~ task_definition: description: | The family and revision (family:revision) or full ARN of the task definition to run. If a revision isn't specified, the latest ACTIVE revision is used. For more info see url to Boto3 docs above. version_added: "8.10" type: string example: executor_task_definition:LATEST default: ~ max_run_task_attempts: description: | The maximum number of times the ECS Executor should attempt to run a task. version_added: "8.10" type: integer example: "3" default: "3" run_task_kwargs: description: | A JSON string containing arguments to provide the ECS `run_task` API (see url above). version_added: "8.10" type: string example: '{"tags": {"key": "schema", "value": "1.0"}}' default: ~ check_health_on_startup: description: | Whether or not to check the ECS Executor health on startup. version_added: "8.11" type: boolean example: "True" default: "True" aws_auth_manager: description: | This section only applies if you are using the AwsAuthManager. In other words, if you set ``[core] auth_manager = airflow.providers.amazon.aws.auth_manager.aws_auth_manager.AwsAuthManager`` in Airflow's configuration. options: conn_id: description: | The Airflow connection (i.e. credentials) used by the AWS auth manager to make API calls to AWS Identity Center and Amazon Verified Permissions. version_added: 8.12.0 type: string example: "aws_default" default: "aws_default" region_name: description: | The name of the AWS Region where Amazon Verified Permissions is configured. Required. version_added: "8.10" type: string example: "us-east-1" default: ~ saml_metadata_url: description: | SAML metadata XML file provided by AWS Identity Center. This URL can be found in the AWS Identity Center console. Required. version_added: 8.12.0 type: string example: "https://portal.sso.<region>.amazonaws.com/saml/metadata/XXXXXXXXXX" default: ~ avp_policy_store_id: description: | Amazon Verified Permissions' policy store ID where all the policies defining user permissions in Airflow are stored. Required. version_added: 8.12.0 type: string example: ~ default: ~ executors: - airflow.providers.amazon.aws.executors.ecs.ecs_executor.AwsEcsExecutor auth-managers: - airflow.providers.amazon.aws.auth_manager.aws_auth_manager.AwsAuthManager queues: - airflow.providers.amazon.aws.queues.sqs.SqsMessageQueueProvider