cfn/classification_pipeline.yaml [1:288]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
AWSTemplateFormatVersion: "2010-09-09"
Description: "Training/Inference Pipeline"

Parameters:
  AthenaDatabaseName:
    Description: "Which Athena Database to Use?"
    Type: String
    Default: "churnblog"

  PipelineBucketName:
    Description: "Which Bucket is Used for the Training/Inference Pipelines?"
    Type: String
    Default: "train-inference-pipeline"

  MakeSegments:
    Description: "Should clustering segemtnation be run in the pipeline?"
    Type: String
    Default: "True"

Mappings:
  AWSRegionArch2DebuggerImageRegistry:
    af-south-1:
      id: 314341159256
    ap-east-1:
      id: 199566480951
    ap-northeast-1:
      id: 430734990657
    ap-northeast-2:
      id: 578805364391
    ap-south-1:
      id: 904829902805
    ap-southeast-1:
      id: 972752614525
    ap-southeast-2:
      id: 184798709955
    ca-central-1:
      id: 519511493484
    cn-north-1:
      id: 618459771430
    cn-northwest-1:
      id: 658757709296
    eu-central-1:
      id: 482524230118
    eu-north-1:
      id: 314864569078
    eu-south-1:
      id: 563282790590
    eu-west-1:
      id: 929884845733
    eu-west-2:
      id: 250201462417
    eu-west-3:
      id: 447278800020
    me-south-1:
      id: 986000313247
    sa-east-1:
      id: 818342061345
    us-east-1:
      id: 503895931360
    us-east-2:
      id: 915447279597
    us-west-1:
      id: 685455198987
    us-west-2:
      id: 895741380848
    us-gov-west-1:
      id: 515509971035

  AWSRegionArch2XGBoostImageRegistry:
    us-east-1:
      id: 683313688378
    us-east-2:
      id: 257758044811
    us-west-1:
      id: 746614075791
    us-west-2:
      id: 246618743249
    ca-central-1:
      id: 341280168497
    eu-west-1:
      id: 141502667606
    eu-west-2:
      id: 764974769150
    eu-central-1:
      id: 492215442770
    eu-west-3:
      id: 659782779980
    eu-north-1:
      id: 662702820516
    ap-east-1:
      id: 651117190479
    ap-northeast-1:
      id: 354813040037
    ap-northeast-2:
      id: 366743142698
    ap-southeast-1:
      id: 121021644041
    ap-southeast-2:
      id: 783357654285
    ap-south-1:
      id: 720646828776
    me-south-1:
      id: 801668240914
    sa-east-1:
      id: 737474898029
    us-gov-west-1:
      id: 414596584902
    us-iso-east-1:
      id: 833128469047
    cn-north-1:
      id: 450853457545
    cn-northwest-1:
      id: 451049120500

Resources:
  IAMRoleStepFunction:
    Type: "AWS::IAM::Role"
    Properties:
      Path: "/"
      RoleName: "AmazonSageMaker-StepFunctionsWorkflowExecutionRole-TIP"
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - states.amazonaws.com
            Action:
              - "sts:AssumeRole"
      MaxSessionDuration: 3600
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaRole"
        - "arn:aws:iam::aws:policy/CloudWatchEventsFullAccess"
        - !Ref IAMManagedPolicyStepFunction
      Description: "Allows Step Functions to access AWS resources on your behalf."

  IAMManagedPolicyS3Access:
    Type: "AWS::IAM::ManagedPolicy"
    Properties:
      ManagedPolicyName: "AmazonSageMaker-ExecutionPolicy-TrainingInferencePipeline"
      Path: "/service-role/"
      PolicyDocument: |
        {
            "Version": "2012-10-17",
            "Statement": [
                {
                    "Effect": "Allow",
                    "Action": [
                        "s3:GetBucketLocation",
                        "s3:GetObject",
                        "s3:ListBucket",
                        "s3:ListBucketMultipartUploads",
                        "s3:ListMultipartUploadParts",
                        "s3:AbortMultipartUpload",
                        "s3:DeleteObject",
                        "s3:CreateBucket",
                        "s3:PutObject"
                    ],
                    "Resource": [
                        "arn:aws:s3:::*"
                    ]
                }
            ]
        }

  IAMRoleSageMaker:
    Type: "AWS::IAM::Role"
    Properties:
      Path: "/service-role/"
      RoleName: "AmazonSageMaker-ExecutionRole-TrainingInferencePipeline"
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - sagemaker.amazonaws.com
            Action:
              - "sts:AssumeRole"
      MaxSessionDuration: 3600
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
        - "arn:aws:iam::aws:policy/AWSStepFunctionsFullAccess"
        - "arn:aws:iam::aws:policy/AmazonSageMakerFullAccess"
        - !Ref IAMManagedPolicyS3Access
      Description: "SageMaker execution role created from the SageMaker AWS Management Console."

  IAMManagedPolicyStepFunction:
    Type: "AWS::IAM::ManagedPolicy"
    Properties:
      ManagedPolicyName: "AmazonSageMaker-StepFunctionsWorkflowExecutionPolicy-TrainingInferencePipeline"
      Path: "/"
      PolicyDocument: !Sub |
        {
            "Version": "2012-10-17",
            "Statement": [
                {
                    "Sid": "VisualEditor0",
                    "Effect": "Allow",
                    "Action": [
                        "events:PutTargets",
                        "events:DescribeRule",
                        "events:PutRule"
                    ],
                    "Resource": [
                        "arn:aws:events:*:*:rule/StepFunctionsGetEventsForSageMakerTrainingJobsRule",
                        "arn:aws:events:*:*:rule/StepFunctionsGetEventsForSageMakerTransformJobsRule",
                        "arn:aws:events:*:*:rule/StepFunctionsGetEventsForSageMakerTuningJobsRule",
                        "arn:aws:events:*:*:rule/StepFunctionsGetEventsForECSTaskRule",
                        "arn:aws:events:*:*:rule/StepFunctionsGetEventsForBatchJobsRule"
                    ]
                },
                {
                    "Sid": "VisualEditor1",
                    "Effect": "Allow",
                    "Action": "iam:PassRole",
                    "Resource": "arn:aws:iam::${AWS::AccountId}:role/service-role/${IAMRoleSageMaker}",
                    "Condition": {
                        "StringEquals": {
                            "iam:PassedToService": "sagemaker.amazonaws.com"
                        }
                    }
                },
                {
                    "Sid": "VisualEditor2",
                    "Effect": "Allow",
                    "Action": [
                        "batch:DescribeJobs",
                        "batch:SubmitJob",
                        "batch:TerminateJob",
                        "dynamodb:DeleteItem",
                        "dynamodb:GetItem",
                        "dynamodb:PutItem",
                        "dynamodb:UpdateItem",
                        "ecs:DescribeTasks",
                        "ecs:RunTask",
                        "ecs:StopTask",
                        "glue:BatchStopJobRun",
                        "glue:GetJobRun",
                        "glue:GetJobRuns",
                        "glue:StartJobRun",
                        "lambda:InvokeFunction",
                        "sagemaker:CreateEndpoint",
                        "sagemaker:CreateEndpointConfig",
                        "sagemaker:CreateHyperParameterTuningJob",
                        "sagemaker:CreateModel",
                        "sagemaker:CreateProcessingJob",
                        "sagemaker:CreateTrainingJob",
                        "sagemaker:CreateTransformJob",
                        "sagemaker:DeleteEndpoint",
                        "sagemaker:DeleteEndpointConfig",
                        "sagemaker:DescribeHyperParameterTuningJob",
                        "sagemaker:DescribeProcessingJob",
                        "sagemaker:DescribeTrainingJob",
                        "sagemaker:DescribeTransformJob",
                        "sagemaker:ListProcessingJobs",
                        "sagemaker:ListTags",
                        "sagemaker:StopHyperParameterTuningJob",
                        "sagemaker:StopProcessingJob",
                        "sagemaker:StopTrainingJob",
                        "sagemaker:StopTransformJob",
                        "sagemaker:UpdateEndpoint",
                        "sns:Publish",
                        "sqs:SendMessage"
                    ],
                    "Resource": "*"
                }
            ]
        }

  StepFunctionsStateMachineTraining:
    Type: "AWS::StepFunctions::StateMachine"
    Properties:
      StateMachineName: "SageMakerPreproTrainWorkflow"
      DefinitionString: !Sub
        - |
          {
            "StartAt": "SageMaker Training Step Preprocessing",
            "States": {
                "SageMaker Training Step Preprocessing": {
                    "Resource": "arn:aws:states:::sagemaker:createProcessingJob.sync",
                    "Parameters": {
                        "ProcessingJobName.$": "$$.Execution.Input['PreprocessingJobName']",
                        "ProcessingInputs": [
                            {
                                "InputName": "code",
                                "AppManaged": false,
                                "S3Input": {
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



cfn/time_to_event_pipeline.yaml [1:288]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
AWSTemplateFormatVersion: "2010-09-09"
Description: "Training/Inference Pipeline"

Parameters:
  AthenaDatabaseName:
    Description: "Which Athena Database to Use?"
    Type: String
    Default: "churnblog"

  PipelineBucketName:
    Description: "Which Bucket is Used for the Training/Inference Pipelines?"
    Type: String
    Default: "train-inference-pipeline"

  MakeSegments:
    Description: "Should clustering segemtnation be run in the pipeline?"
    Type: String
    Default: "True"

Mappings:
  AWSRegionArch2DebuggerImageRegistry:
    af-south-1:
      id: 314341159256
    ap-east-1:
      id: 199566480951
    ap-northeast-1:
      id: 430734990657
    ap-northeast-2:
      id: 578805364391
    ap-south-1:
      id: 904829902805
    ap-southeast-1:
      id: 972752614525
    ap-southeast-2:
      id: 184798709955
    ca-central-1:
      id: 519511493484
    cn-north-1:
      id: 618459771430
    cn-northwest-1:
      id: 658757709296
    eu-central-1:
      id: 482524230118
    eu-north-1:
      id: 314864569078
    eu-south-1:
      id: 563282790590
    eu-west-1:
      id: 929884845733
    eu-west-2:
      id: 250201462417
    eu-west-3:
      id: 447278800020
    me-south-1:
      id: 986000313247
    sa-east-1:
      id: 818342061345
    us-east-1:
      id: 503895931360
    us-east-2:
      id: 915447279597
    us-west-1:
      id: 685455198987
    us-west-2:
      id: 895741380848
    us-gov-west-1:
      id: 515509971035

  AWSRegionArch2XGBoostImageRegistry:
    us-east-1:
      id: 683313688378
    us-east-2:
      id: 257758044811
    us-west-1:
      id: 746614075791
    us-west-2:
      id: 246618743249
    ca-central-1:
      id: 341280168497
    eu-west-1:
      id: 141502667606
    eu-west-2:
      id: 764974769150
    eu-central-1:
      id: 492215442770
    eu-west-3:
      id: 659782779980
    eu-north-1:
      id: 662702820516
    ap-east-1:
      id: 651117190479
    ap-northeast-1:
      id: 354813040037
    ap-northeast-2:
      id: 366743142698
    ap-southeast-1:
      id: 121021644041
    ap-southeast-2:
      id: 783357654285
    ap-south-1:
      id: 720646828776
    me-south-1:
      id: 801668240914
    sa-east-1:
      id: 737474898029
    us-gov-west-1:
      id: 414596584902
    us-iso-east-1:
      id: 833128469047
    cn-north-1:
      id: 450853457545
    cn-northwest-1:
      id: 451049120500

Resources:
  IAMRoleStepFunction:
    Type: "AWS::IAM::Role"
    Properties:
      Path: "/"
      RoleName: "AmazonSageMaker-StepFunctionsWorkflowExecutionRole-TIP"
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - states.amazonaws.com
            Action:
              - "sts:AssumeRole"
      MaxSessionDuration: 3600
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaRole"
        - "arn:aws:iam::aws:policy/CloudWatchEventsFullAccess"
        - !Ref IAMManagedPolicyStepFunction
      Description: "Allows Step Functions to access AWS resources on your behalf."

  IAMManagedPolicyS3Access:
    Type: "AWS::IAM::ManagedPolicy"
    Properties:
      ManagedPolicyName: "AmazonSageMaker-ExecutionPolicy-TrainingInferencePipeline"
      Path: "/service-role/"
      PolicyDocument: |
        {
            "Version": "2012-10-17",
            "Statement": [
                {
                    "Effect": "Allow",
                    "Action": [
                        "s3:GetBucketLocation",
                        "s3:GetObject",
                        "s3:ListBucket",
                        "s3:ListBucketMultipartUploads",
                        "s3:ListMultipartUploadParts",
                        "s3:AbortMultipartUpload",
                        "s3:DeleteObject",
                        "s3:CreateBucket",
                        "s3:PutObject"
                    ],
                    "Resource": [
                        "arn:aws:s3:::*"
                    ]
                }
            ]
        }

  IAMRoleSageMaker:
    Type: "AWS::IAM::Role"
    Properties:
      Path: "/service-role/"
      RoleName: "AmazonSageMaker-ExecutionRole-TrainingInferencePipeline"
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - sagemaker.amazonaws.com
            Action:
              - "sts:AssumeRole"
      MaxSessionDuration: 3600
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
        - "arn:aws:iam::aws:policy/AWSStepFunctionsFullAccess"
        - "arn:aws:iam::aws:policy/AmazonSageMakerFullAccess"
        - !Ref IAMManagedPolicyS3Access
      Description: "SageMaker execution role created from the SageMaker AWS Management Console."

  IAMManagedPolicyStepFunction:
    Type: "AWS::IAM::ManagedPolicy"
    Properties:
      ManagedPolicyName: "AmazonSageMaker-StepFunctionsWorkflowExecutionPolicy-TrainingInferencePipeline"
      Path: "/"
      PolicyDocument: !Sub |
        {
            "Version": "2012-10-17",
            "Statement": [
                {
                    "Sid": "VisualEditor0",
                    "Effect": "Allow",
                    "Action": [
                        "events:PutTargets",
                        "events:DescribeRule",
                        "events:PutRule"
                    ],
                    "Resource": [
                        "arn:aws:events:*:*:rule/StepFunctionsGetEventsForSageMakerTrainingJobsRule",
                        "arn:aws:events:*:*:rule/StepFunctionsGetEventsForSageMakerTransformJobsRule",
                        "arn:aws:events:*:*:rule/StepFunctionsGetEventsForSageMakerTuningJobsRule",
                        "arn:aws:events:*:*:rule/StepFunctionsGetEventsForECSTaskRule",
                        "arn:aws:events:*:*:rule/StepFunctionsGetEventsForBatchJobsRule"
                    ]
                },
                {
                    "Sid": "VisualEditor1",
                    "Effect": "Allow",
                    "Action": "iam:PassRole",
                    "Resource": "arn:aws:iam::${AWS::AccountId}:role/service-role/${IAMRoleSageMaker}",
                    "Condition": {
                        "StringEquals": {
                            "iam:PassedToService": "sagemaker.amazonaws.com"
                        }
                    }
                },
                {
                    "Sid": "VisualEditor2",
                    "Effect": "Allow",
                    "Action": [
                        "batch:DescribeJobs",
                        "batch:SubmitJob",
                        "batch:TerminateJob",
                        "dynamodb:DeleteItem",
                        "dynamodb:GetItem",
                        "dynamodb:PutItem",
                        "dynamodb:UpdateItem",
                        "ecs:DescribeTasks",
                        "ecs:RunTask",
                        "ecs:StopTask",
                        "glue:BatchStopJobRun",
                        "glue:GetJobRun",
                        "glue:GetJobRuns",
                        "glue:StartJobRun",
                        "lambda:InvokeFunction",
                        "sagemaker:CreateEndpoint",
                        "sagemaker:CreateEndpointConfig",
                        "sagemaker:CreateHyperParameterTuningJob",
                        "sagemaker:CreateModel",
                        "sagemaker:CreateProcessingJob",
                        "sagemaker:CreateTrainingJob",
                        "sagemaker:CreateTransformJob",
                        "sagemaker:DeleteEndpoint",
                        "sagemaker:DeleteEndpointConfig",
                        "sagemaker:DescribeHyperParameterTuningJob",
                        "sagemaker:DescribeProcessingJob",
                        "sagemaker:DescribeTrainingJob",
                        "sagemaker:DescribeTransformJob",
                        "sagemaker:ListProcessingJobs",
                        "sagemaker:ListTags",
                        "sagemaker:StopHyperParameterTuningJob",
                        "sagemaker:StopProcessingJob",
                        "sagemaker:StopTrainingJob",
                        "sagemaker:StopTransformJob",
                        "sagemaker:UpdateEndpoint",
                        "sns:Publish",
                        "sqs:SendMessage"
                    ],
                    "Resource": "*"
                }
            ]
        }

  StepFunctionsStateMachineTraining:
    Type: "AWS::StepFunctions::StateMachine"
    Properties:
      StateMachineName: "SageMakerPreproTrainWorkflow"
      DefinitionString: !Sub
        - |
          {
            "StartAt": "SageMaker Training Step Preprocessing",
            "States": {
                "SageMaker Training Step Preprocessing": {
                    "Resource": "arn:aws:states:::sagemaker:createProcessingJob.sync",
                    "Parameters": {
                        "ProcessingJobName.$": "$$.Execution.Input['PreprocessingJobName']",
                        "ProcessingInputs": [
                            {
                                "InputName": "code",
                                "AppManaged": false,
                                "S3Input": {
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



