cfn/classification_pipeline.yaml [838:1174]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                                "--cluster",
                                "${MakeSegments}"
                            ],
                            "ContainerEntrypoint": [
                                "python3",
                                "/opt/ml/processing/input/code/inferpreprocessing.py"
                            ]
                        },
                        "RoleArn": "arn:aws:iam::${AWS::AccountId}:role/service-role/AmazonSageMaker-ExecutionRole-TrainingInferencePipeline",
                        "ProcessingResources": {
                            "ClusterConfig": {
                                "InstanceCount": 1,
                                "InstanceType": "ml.m5.xlarge",
                                "VolumeSizeInGB": 30
                            }
                        },
                        "StoppingCondition": {
                            "MaxRuntimeInSeconds": 1200
                        }
                    },
                    "Type": "Task",
                    "Next": "Transform Input Dataset",
                    "Catch": [
                        {
                            "ErrorEquals": [
                                "States.TaskFailed"
                            ],
                            "Next": "ML Workflow failed"
                        }
                    ]
                },
                "Transform Input Dataset": {
                    "Resource": "arn:aws:states:::sagemaker:createTransformJob.sync",
                    "Parameters": {
                        "TransformJobName.$": "$$.Execution.Input['InferencingJobName']",
                        "ModelName.$": "$$.Execution.Input['ModelName']",
                        "TransformInput": {
                            "DataSource": {
                                "S3DataSource": {
                                    "S3DataType": "S3Prefix",
                                    "S3Uri": "s3://${PipelineBucketName}/data/intermediate/infer_data"
                                }
                            },
                            "ContentType": "text/csv"
                        },
                        "TransformOutput": {
                            "S3OutputPath": "s3://${PipelineBucketName}/data/inference_result"
                        },
                        "TransformResources": {
                            "InstanceCount": 1,
                            "InstanceType": "ml.m5.large"
                        }
                    },
                    "Type": "Task",
                    "End": true,
                    "Catch": [
                        {
                            "ErrorEquals": [
                                "States.TaskFailed"
                            ],
                            "Next": "ML Workflow failed"
                        }
                    ]
                },
                "ML Workflow failed": {
                    "Cause": "SageMakerProcessingJobFailed",
                    "Type": "Fail"
                }
            }
          }
        - {
            XgbImageID:
              !FindInMap [
                AWSRegionArch2XGBoostImageRegistry,
                !Ref "AWS::Region",
                id,
              ],
          }
      RoleArn: !GetAtt IAMRoleStepFunction.Arn
      StateMachineType: "STANDARD"
      LoggingConfiguration:
        # Destinations:
        # - CloudWatchLogsLogGroup:
        # LogGroupArn: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/vendedlogs/states/SageMakerProcessingInferWorkflow-Log:*"
        IncludeExecutionData: true
        Level: "OFF"

  LambdaFunctionGetModelData:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: "getSageMakerModelUrl"
      Runtime: python3.6
      Timeout: 900
      MemorySize: 128
      Role: !GetAtt LambdaIAMRole.Arn
      Handler: index.lambda_handler
      Code:
        ZipFile: !Sub |
          import boto3

          def lambda_handler(event, context):
              sm_client = boto3.client("sagemaker")

              job = sm_client.describe_model(
                  ModelName=event["TrainingJobName"]["ModelArn"].split("/")[-1]
              )
              training_job_name = job["PrimaryContainer"]["ModelDataUrl"].split('/')[-3]

              return {
                  "statusCode": 200,
                  "body": {
                      "modelDataUrl": job["PrimaryContainer"]["ModelDataUrl"],
                      "train_debug_path": f"s3://${PipelineBucketName}/debug/{training_job_name}/"
                  },
              }
      Description: Get the HyperParameter of the Best Tuning Job
      TracingConfig:
        Mode: Active

  LambdaFunctionBestHyper:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: "getSMTuningStepBestHyperParameters"
      Runtime: python3.6
      Timeout: 900
      MemorySize: 128
      Role: !GetAtt LambdaIAMRole.Arn
      Handler: index.lambda_handler
      Code:
        ZipFile: |
          import boto3
          def lambda_handler(event, context):
              sm_client = boto3.client("sagemaker")
              job = sm_client.describe_training_job(TrainingJobName=event['TrainingJobName']['TrainingJobName'])
              hyper_parameters = job['HyperParameters']
              return {
                  'statusCode': 200,
                  'body': {
                  'bestTrainingJobName': event['TrainingJobName']['TrainingJobName'],
                  'bestHyperParameter': hyper_parameters,
                  }
              }
      Description: Get the HyperParameter of the Best Tuning Job
      TracingConfig:
        Mode: Active

  LambdaIAMRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - "sts:AssumeRole"
      Path: /
      MaxSessionDuration: 3600
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/AmazonSageMakerFullAccess"
        - "arn:aws:iam::aws:policy/AWSStepFunctionsFullAccess"
        - "arn:aws:iam::aws:policy/AmazonS3FullAccess"
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
      # Policies:
      #   - PolicyName: allow_lambda_to_log
      #     PolicyDocument:
      #       Version: 2012-10-17
      #       Statement:
      #         - Effect: Allow
      #           Action:
      #             - "logs:CreateLogGroup"
      #             - "logs:CreateLogStream"
      #             - "logs:PutLogEvents"
      #           Resource:
      #             - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/getSMTuningStepBestHyperParameters:*"
      #             - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/getSageMakerModelUrl:*"

  LambdaFunctionInvokeTrain:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: "invokeTrainingStepFunction"
      Runtime: python3.6
      Timeout: 900
      MemorySize: 128
      Role: !GetAtt LambdaIAMRole.Arn
      Handler: index.lambda_handler
      Code:
        ZipFile: !Sub |
          import json
          import boto3
          import pprint
          import uuid

          #initialize
          pp = pprint.PrettyPrinter(indent=4)

          def lambda_handler(event, context):
              '''
              triggers a step function from the S3 landing event,
              passing along the S3 file info
              '''

              #Fallback tests for initializations outside scope
              try:
                  pp
              except NameError:
                  pp = pprint.PrettyPrinter(indent=4)

              training_job_name = "scikit-learn-training-{}".format(
                  uuid.uuid1().hex,
              )  # Each Training Job requires a unique name
              preprocessing_job_name = "scikit-learn-sm-preprocessing-{}".format(
                  uuid.uuid1().hex,
              )  # Each Preprocessing job requires a unique name,
              hyperparametertunning_job_name = f"scikit-learn-tune-{uuid.uuid1().hex}"[
                  :30
              ]  # Each Preprocessing job requires a unique name,

              model_name = f"scikit-learn-model-{uuid.uuid1().hex}"

              evaluation_job_name = "scikit-learn-sm-evaluation-{}".format(
                  uuid.uuid1().hex,
              )  # Each Evaluation Job requires a unique name

              input= {
                  "PreprocessingJobName": preprocessing_job_name,
                  "TuningJobName": hyperparametertunning_job_name,
                  "TrainingJobName": training_job_name,
                  "ModelName": model_name,
                  "EvaluationProcessingJobName": evaluation_job_name,
              }

              stepFunction = boto3.client('stepfunctions')
              response = stepFunction.start_execution(
                  stateMachineArn='arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:SageMakerPreproTrainWorkflow',
                  input = json.dumps(input, indent=4)
              )

              s3 = boto3.resource("s3")
              s3.Bucket("${PipelineBucketName}").put_object(
                  Key="model/ledger/model_name.txt",
                  Body=model_name.encode("utf-8")
                )
              return pp.pprint(response)
      Description: Invoke Training Step Function Pipeline
      TracingConfig:
        Mode: Active

  LambdaFunctionInvokeInfer:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: "invokeInferStepFunction"
      Runtime: python3.6
      Timeout: 900
      MemorySize: 128
      Role: !GetAtt LambdaIAMRole.Arn
      Handler: index.lambda_handler
      Code:
        ZipFile: !Sub |
          import json
          import boto3
          import pprint
          import uuid

          #initialize
          pp = pprint.PrettyPrinter(indent=4)

          def lambda_handler(event, context):
              '''
              triggers a step function from the S3 landing event,
              passing along the S3 file info
              '''

              #Fallback tests for initializations outside scope
              try:
                  pp
              except NameError:
                  pp = pprint.PrettyPrinter(indent=4)

              # Generate unique names for Preprocessing Job, Training Job,
              # and Model Evaluation Job for the Step Functions Workflow
              inference_job_preproc_name = "scikit-learn-infer-preprocessing-{}".format(
                  uuid.uuid1().hex,
              )[
                  :60
              ]  # Each Training Job requires a unique name

              inference_job_predict_name = "scikit-learn-infer-predict-{}".format(
                  uuid.uuid1().hex,
              )  # Each Training Job requires a unique name


              s3 = boto3.resource('s3')
              obj = s3.Object("${PipelineBucketName}", "model/ledger/model_name.txt")
              model_name = obj.get()['Body'].read().decode("utf-8")
              print(model_name)

              input= {
                  "PreprocessingJobName": inference_job_preproc_name,
                  "InferencingJobName": inference_job_predict_name,
                  "ModelName": model_name,
              }

              stepFunction = boto3.client('stepfunctions')
              response = stepFunction.start_execution(
                  stateMachineArn='arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:SageMakerPreproInferWorkflow',
                  input = json.dumps(input, indent=4)
              )

              return pp.pprint(response)
      Description: Invoke Inference Step Function Pipeline
      TracingConfig:
        Mode: Active

Outputs:
  StepFunctionTrainingPipeline:
    Description: The StepFunction Pipeline for Training
    Value: !GetAtt StepFunctionsStateMachineTraining.Arn
    Export:
      Name: StepFunctionTrainingPipeline
  StepFunctionInferencePipeline:
    Description: The StepFunction Pipeline for Inference
    Value: !GetAtt StepFunctionsStateMachineInference.Arn
    Export:
      Name: StepFunctionInferencePipeline
  LambdaFunctionTriggerTrain:
    Description: The Lambda Trigger Training Pipeline
    Value: !GetAtt LambdaFunctionInvokeTrain.Arn
    Export:
      Name: LambdaFunctionTriggerTraining
  LambdaFunctionTriggerInfer:
    Description: The Lambda Trigger Inference Pipeline
    Value: !GetAtt LambdaFunctionInvokeInfer.Arn
    Export:
      Name: LambdaFunctionTriggerInference
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



cfn/time_to_event_pipeline.yaml [841:1177]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                                "--cluster",
                                "${MakeSegments}"
                            ],
                            "ContainerEntrypoint": [
                                "python3",
                                "/opt/ml/processing/input/code/inferpreprocessing.py"
                            ]
                        },
                        "RoleArn": "arn:aws:iam::${AWS::AccountId}:role/service-role/AmazonSageMaker-ExecutionRole-TrainingInferencePipeline",
                        "ProcessingResources": {
                            "ClusterConfig": {
                                "InstanceCount": 1,
                                "InstanceType": "ml.m5.xlarge",
                                "VolumeSizeInGB": 30
                            }
                        },
                        "StoppingCondition": {
                            "MaxRuntimeInSeconds": 1200
                        }
                    },
                    "Type": "Task",
                    "Next": "Transform Input Dataset",
                    "Catch": [
                        {
                            "ErrorEquals": [
                                "States.TaskFailed"
                            ],
                            "Next": "ML Workflow failed"
                        }
                    ]
                },
                "Transform Input Dataset": {
                    "Resource": "arn:aws:states:::sagemaker:createTransformJob.sync",
                    "Parameters": {
                        "TransformJobName.$": "$$.Execution.Input['InferencingJobName']",
                        "ModelName.$": "$$.Execution.Input['ModelName']",
                        "TransformInput": {
                            "DataSource": {
                                "S3DataSource": {
                                    "S3DataType": "S3Prefix",
                                    "S3Uri": "s3://${PipelineBucketName}/data/intermediate/infer_data"
                                }
                            },
                            "ContentType": "text/csv"
                        },
                        "TransformOutput": {
                            "S3OutputPath": "s3://${PipelineBucketName}/data/inference_result"
                        },
                        "TransformResources": {
                            "InstanceCount": 1,
                            "InstanceType": "ml.m5.large"
                        }
                    },
                    "Type": "Task",
                    "End": true,
                    "Catch": [
                        {
                            "ErrorEquals": [
                                "States.TaskFailed"
                            ],
                            "Next": "ML Workflow failed"
                        }
                    ]
                },
                "ML Workflow failed": {
                    "Cause": "SageMakerProcessingJobFailed",
                    "Type": "Fail"
                }
            }
          }
        - {
            XgbImageID:
              !FindInMap [
                AWSRegionArch2XGBoostImageRegistry,
                !Ref "AWS::Region",
                id,
              ],
          }
      RoleArn: !GetAtt IAMRoleStepFunction.Arn
      StateMachineType: "STANDARD"
      LoggingConfiguration:
        # Destinations:
        # - CloudWatchLogsLogGroup:
        # LogGroupArn: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/vendedlogs/states/SageMakerProcessingInferWorkflow-Log:*"
        IncludeExecutionData: true
        Level: "OFF"

  LambdaFunctionGetModelData:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: "getSageMakerModelUrl"
      Runtime: python3.6
      Timeout: 900
      MemorySize: 128
      Role: !GetAtt LambdaIAMRole.Arn
      Handler: index.lambda_handler
      Code:
        ZipFile: !Sub |
          import boto3

          def lambda_handler(event, context):
              sm_client = boto3.client("sagemaker")

              job = sm_client.describe_model(
                  ModelName=event["TrainingJobName"]["ModelArn"].split("/")[-1]
              )
              training_job_name = job["PrimaryContainer"]["ModelDataUrl"].split('/')[-3]

              return {
                  "statusCode": 200,
                  "body": {
                      "modelDataUrl": job["PrimaryContainer"]["ModelDataUrl"],
                      "train_debug_path": f"s3://${PipelineBucketName}/debug/{training_job_name}/"
                  },
              }
      Description: Get the HyperParameter of the Best Tuning Job
      TracingConfig:
        Mode: Active

  LambdaFunctionBestHyper:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: "getSMTuningStepBestHyperParameters"
      Runtime: python3.6
      Timeout: 900
      MemorySize: 128
      Role: !GetAtt LambdaIAMRole.Arn
      Handler: index.lambda_handler
      Code:
        ZipFile: |
          import boto3
          def lambda_handler(event, context):
              sm_client = boto3.client("sagemaker")
              job = sm_client.describe_training_job(TrainingJobName=event['TrainingJobName']['TrainingJobName'])
              hyper_parameters = job['HyperParameters']
              return {
                  'statusCode': 200,
                  'body': {
                  'bestTrainingJobName': event['TrainingJobName']['TrainingJobName'],
                  'bestHyperParameter': hyper_parameters,
                  }
              }
      Description: Get the HyperParameter of the Best Tuning Job
      TracingConfig:
        Mode: Active

  LambdaIAMRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - "sts:AssumeRole"
      Path: /
      MaxSessionDuration: 3600
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/AmazonSageMakerFullAccess"
        - "arn:aws:iam::aws:policy/AWSStepFunctionsFullAccess"
        - "arn:aws:iam::aws:policy/AmazonS3FullAccess"
        - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
      # Policies:
      #   - PolicyName: allow_lambda_to_log
      #     PolicyDocument:
      #       Version: 2012-10-17
      #       Statement:
      #         - Effect: Allow
      #           Action:
      #             - "logs:CreateLogGroup"
      #             - "logs:CreateLogStream"
      #             - "logs:PutLogEvents"
      #           Resource:
      #             - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/getSMTuningStepBestHyperParameters:*"
      #             - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/getSageMakerModelUrl:*"

  LambdaFunctionInvokeTrain:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: "invokeTrainingStepFunction"
      Runtime: python3.6
      Timeout: 900
      MemorySize: 128
      Role: !GetAtt LambdaIAMRole.Arn
      Handler: index.lambda_handler
      Code:
        ZipFile: !Sub |
          import json
          import boto3
          import pprint
          import uuid

          #initialize
          pp = pprint.PrettyPrinter(indent=4)

          def lambda_handler(event, context):
              '''
              triggers a step function from the S3 landing event,
              passing along the S3 file info
              '''

              #Fallback tests for initializations outside scope
              try:
                  pp
              except NameError:
                  pp = pprint.PrettyPrinter(indent=4)

              training_job_name = "scikit-learn-training-{}".format(
                  uuid.uuid1().hex,
              )  # Each Training Job requires a unique name
              preprocessing_job_name = "scikit-learn-sm-preprocessing-{}".format(
                  uuid.uuid1().hex,
              )  # Each Preprocessing job requires a unique name,
              hyperparametertunning_job_name = f"scikit-learn-tune-{uuid.uuid1().hex}"[
                  :30
              ]  # Each Preprocessing job requires a unique name,

              model_name = f"scikit-learn-model-{uuid.uuid1().hex}"

              evaluation_job_name = "scikit-learn-sm-evaluation-{}".format(
                  uuid.uuid1().hex,
              )  # Each Evaluation Job requires a unique name

              input= {
                  "PreprocessingJobName": preprocessing_job_name,
                  "TuningJobName": hyperparametertunning_job_name,
                  "TrainingJobName": training_job_name,
                  "ModelName": model_name,
                  "EvaluationProcessingJobName": evaluation_job_name,
              }

              stepFunction = boto3.client('stepfunctions')
              response = stepFunction.start_execution(
                  stateMachineArn='arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:SageMakerPreproTrainWorkflow',
                  input = json.dumps(input, indent=4)
              )

              s3 = boto3.resource("s3")
              s3.Bucket("${PipelineBucketName}").put_object(
                  Key="model/ledger/model_name.txt",
                  Body=model_name.encode("utf-8")
                )
              return pp.pprint(response)
      Description: Invoke Training Step Function Pipeline
      TracingConfig:
        Mode: Active

  LambdaFunctionInvokeInfer:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: "invokeInferStepFunction"
      Runtime: python3.6
      Timeout: 900
      MemorySize: 128
      Role: !GetAtt LambdaIAMRole.Arn
      Handler: index.lambda_handler
      Code:
        ZipFile: !Sub |
          import json
          import boto3
          import pprint
          import uuid

          #initialize
          pp = pprint.PrettyPrinter(indent=4)

          def lambda_handler(event, context):
              '''
              triggers a step function from the S3 landing event,
              passing along the S3 file info
              '''

              #Fallback tests for initializations outside scope
              try:
                  pp
              except NameError:
                  pp = pprint.PrettyPrinter(indent=4)

              # Generate unique names for Preprocessing Job, Training Job,
              # and Model Evaluation Job for the Step Functions Workflow
              inference_job_preproc_name = "scikit-learn-infer-preprocessing-{}".format(
                  uuid.uuid1().hex,
              )[
                  :60
              ]  # Each Training Job requires a unique name

              inference_job_predict_name = "scikit-learn-infer-predict-{}".format(
                  uuid.uuid1().hex,
              )  # Each Training Job requires a unique name


              s3 = boto3.resource('s3')
              obj = s3.Object("${PipelineBucketName}", "model/ledger/model_name.txt")
              model_name = obj.get()['Body'].read().decode("utf-8")
              print(model_name)

              input= {
                  "PreprocessingJobName": inference_job_preproc_name,
                  "InferencingJobName": inference_job_predict_name,
                  "ModelName": model_name,
              }

              stepFunction = boto3.client('stepfunctions')
              response = stepFunction.start_execution(
                  stateMachineArn='arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:SageMakerPreproInferWorkflow',
                  input = json.dumps(input, indent=4)
              )

              return pp.pprint(response)
      Description: Invoke Inference Step Function Pipeline
      TracingConfig:
        Mode: Active

Outputs:
  StepFunctionTrainingPipeline:
    Description: The StepFunction Pipeline for Training
    Value: !GetAtt StepFunctionsStateMachineTraining.Arn
    Export:
      Name: StepFunctionTrainingPipeline
  StepFunctionInferencePipeline:
    Description: The StepFunction Pipeline for Inference
    Value: !GetAtt StepFunctionsStateMachineInference.Arn
    Export:
      Name: StepFunctionInferencePipeline
  LambdaFunctionTriggerTrain:
    Description: The Lambda Trigger Training Pipeline
    Value: !GetAtt LambdaFunctionInvokeTrain.Arn
    Export:
      Name: LambdaFunctionTriggerTraining
  LambdaFunctionTriggerInfer:
    Description: The Lambda Trigger Inference Pipeline
    Value: !GetAtt LambdaFunctionInvokeInfer.Arn
    Export:
      Name: LambdaFunctionTriggerInference
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



