template/template.yaml [285:496]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      ServiceRole:
        Ref: BatchServiceRole
  BatchProcessS3Bucket:
    Type: AWS::S3::Bucket
    DependsOn: BatchProcessBucketPermission
    Properties: 
      BucketName: 
          !Sub '${StackName}-${AWS::AccountId}'
      NotificationConfiguration:
        LambdaConfigurations:
        - Event: 's3:ObjectCreated:*'
          Function: !GetAtt BatchProcessingLambdaInvokeFunction.Arn
  BatchProcessBucketPermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: 'lambda:InvokeFunction'
      FunctionName: !Ref BatchProcessingLambdaInvokeFunction
      Principal: s3.amazonaws.com
      SourceAccount: !Ref "AWS::AccountId"
      SourceArn: !Sub "arn:aws:s3:::${StackName}-${AWS::AccountId}"
  BatchProcessingLambdaInvokeFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: !Sub "${StackName}-lambda"
      Description: Python Function Handler that would be triggered BY s3 events TO the aws batch
      Handler: index.lambda_handler
      Runtime: python3.6
      MemorySize: 128
      Timeout: 30
      Environment:
        Variables:
          BATCH_JOB_QUEUE_NAME: !Sub "${StackName}-queue"
          DYNAMO_TABLE_NAME: !Sub ${StackName}
      Role:
        Fn::GetAtt:
          - LambdaExecutionRole
          - Arn
      Code:
        ZipFile: |
          import os
          import json
          import boto3

          def lambda_handler(event, context):
              inputFileName = ""
              bucketName = ""

              for record in event['Records']:
                bucketName = record['s3']['bucket']['name']
                inputFileName = record['s3']['object']['key'] 

              response = {
                  'statusCode': 200,
                  'body': json.dumps('Input Received - ' + json.dumps(event))
              }

              batch = boto3.client('batch')
              region = batch.meta.region_name

              batch_job_queue_name = os.environ["BATCH_JOB_QUEUE_NAME"]
              db_table_name  = os.environ["DYNAMO_TABLE_NAME"]

              batchCommand = "--bucketName " + bucketName  + " --fileName " + inputFileName + " --region " + region + " --dbTableName " + db_table_name

              out = "inputFileName - " + bucketName + "/" + inputFileName + " Region " + region + " DBTableName " + db_table_name 
              out = out + "  " + batchCommand
              print(out)

              response = batch.submit_job(jobName=batch_job_queue_name, 
                                          jobQueue=batch_job_queue_name, 
                                          jobDefinition='BatchJobDefinition', 
                                          containerOverrides={
                                              "command": [ "python", "batch_processor.py", batchCommand  ],
                                              "environment": [ 
                                                  {"name": "InputBucket", "value": bucketName},
                                                  {"name": "FileName", "value": inputFileName},
                                                  {"name": "Region", "value": region},
                                                  {"name": "DBTableName", "value": db_table_name}
                                              ]
                                          })

              print("Job ID is {}.".format(response['jobId']))
              return response  
  #Code Commit
  CodeCommitRepository:
    Type: AWS::CodeCommit::Repository
    Properties:
      RepositoryName: !Sub ${StackName}-repo
      RepositoryDescription: Respository to maintain code related to the Batch Processing Jobs.
  #Code Build
  CodeBuildProject:
    Type: AWS::CodeBuild::Project
    Properties:
      Name: !Sub ${StackName}-build
      Description: Batch processing application codebuild project.
      ServiceRole: !GetAtt CodeBuildRole.Arn
      Artifacts:
        Type: no_artifacts
      Environment:
        Type: LINUX_CONTAINER
        ComputeType: BUILD_GENERAL1_SMALL
        Image: aws/codebuild/amazonlinux2-x86_64-standard:1.0
        PrivilegedMode: true
        EnvironmentVariables:
            - Name: REPOSITORY_URI
              Type: PLAINTEXT
              Value: 
                Fn::Join:
                - ''
                - - Ref: AWS::AccountId
                  - .dkr.ecr.
                  - Ref: AWS::Region
                  - !Sub '.amazonaws.com/${StackName}-repository:latest'

            - Name: AWS_DEFAULT_REGION
              Type: PLAINTEXT
              Value: 
                Ref: AWS::Region        
      Source:
        BuildSpec: config/buildspec.yml
        Location: 
          Fn::Join:
            - ''
            - - 'https://git-codecommit.'
              - Ref: AWS::Region
              - '.amazonaws.com/v1/repos/'
              - !Sub ${StackName}-repo
        Type: CODECOMMIT 
      SourceVersion: refs/heads/master
      TimeoutInMinutes: 10 
  CodeBuildRole:
    Type: AWS::IAM::Role
    Properties:
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess 
      - arn:aws:iam::aws:policy/AWSCodeCommitFullAccess   
      AssumeRolePolicyDocument:
        Statement:
        - Action: ['sts:AssumeRole']
          Effect: Allow
          Principal:
            Service: [codebuild.amazonaws.com]
        Version: '2012-10-17'
      Path: /
      Policies:
        - PolicyName: CodeBuildAccess
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Action:
                - 'logs:*'
                - 'ec2:CreateNetworkInterface'
                - 'ec2:DescribeNetworkInterfaces'
                - 'ec2:DeleteNetworkInterface'
                - 'ec2:DescribeSubnets'
                - 'ec2:DescribeSecurityGroups'
                - 'ec2:DescribeDhcpOptions'
                - 'ec2:DescribeVpcs'
                - 'ec2:CreateNetworkInterfacePermission'
                Effect: Allow
                Resource: '*'
  # CloudWatchEvents Code build Rold            
  CloudWatchEventsCodeBuildRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub ${StackName}-cw-events-codebuild-role
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          -
            Effect: Allow
            Principal:
              Service:
                - events.amazonaws.com
            Action: sts:AssumeRole
      Policies:
      - PolicyName: aws-events-code-build
        PolicyDocument:
          Version: 2012-10-17
          Statement:
          - Effect: Allow
            Action:
              - 'codebuild:StartBuild'
            Resource: !GetAtt CodeBuildProject.Arn              
  # CloudWatch Event Rule for codecommit build trigger
  CloudWatchEventCodeBuildEventRule: 
    Type: AWS::Events::Rule
    Properties: 
      Description: "This event rule triggers the build on code commit event"
      EventPattern: 
        source: 
          - "aws.codecommit"
        detail-type: 
          - "CodeCommit Repository State Change"
        detail: 
          event: 
            - "referenceCreated"
            - "referenceUpdated"
          referenceType: 
            - "branch"
          referenceName: 
            - "master"                    
      State: "ENABLED"
      Targets: 
        - 
          Arn: {'Fn::GetAtt': [CodeBuildProject, Arn]}
          Id: cloudwatch-codebuild-eventrules  
          RoleArn: !GetAtt CloudWatchEventsCodeBuildRole.Arn
  BatchProcessRepository: 
    Type: AWS::ECR::Repository
    Properties: 
      RepositoryName: !Sub ${StackName}-repository
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



template/template_ec2.yaml [157:368]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      ServiceRole:
        Ref: BatchServiceRole
  BatchProcessS3Bucket:
    Type: AWS::S3::Bucket
    DependsOn: BatchProcessBucketPermission
    Properties: 
      BucketName: 
          !Sub '${StackName}-${AWS::AccountId}'
      NotificationConfiguration:
        LambdaConfigurations:
        - Event: 's3:ObjectCreated:*'
          Function: !GetAtt BatchProcessingLambdaInvokeFunction.Arn
  BatchProcessBucketPermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: 'lambda:InvokeFunction'
      FunctionName: !Ref BatchProcessingLambdaInvokeFunction
      Principal: s3.amazonaws.com
      SourceAccount: !Ref "AWS::AccountId"
      SourceArn: !Sub "arn:aws:s3:::${StackName}-${AWS::AccountId}"
  BatchProcessingLambdaInvokeFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: !Sub "${StackName}-lambda"
      Description: Python Function Handler that would be triggered BY s3 events TO the aws batch
      Handler: index.lambda_handler
      Runtime: python3.6
      MemorySize: 128
      Timeout: 30
      Environment:
        Variables:
          BATCH_JOB_QUEUE_NAME: !Sub "${StackName}-queue"
          DYNAMO_TABLE_NAME: !Sub ${StackName}      
      Role:
        Fn::GetAtt:
          - LambdaExecutionRole
          - Arn
      Code:
        ZipFile: |
          import os
          import json
          import boto3

          def lambda_handler(event, context):
              inputFileName = ""
              bucketName = ""

              for record in event['Records']:
                bucketName = record['s3']['bucket']['name']
                inputFileName = record['s3']['object']['key'] 

              response = {
                  'statusCode': 200,
                  'body': json.dumps('Input Received - ' + json.dumps(event))
              }

              batch = boto3.client('batch')
              region = batch.meta.region_name

              batch_job_queue_name = os.environ["BATCH_JOB_QUEUE_NAME"]
              db_table_name  = os.environ["DYNAMO_TABLE_NAME"]

              batchCommand = "--bucketName " + bucketName  + " --fileName " + inputFileName + " --region " + region + " --dbTableName " + db_table_name

              out = "inputFileName - " + bucketName + "/" + inputFileName + " Region " + region + " DBTableName " + db_table_name 
              out = out + "  " + batchCommand
              print(out)

              response = batch.submit_job(jobName=batch_job_queue_name, 
                                          jobQueue=batch_job_queue_name, 
                                          jobDefinition='BatchJobDefinition', 
                                          containerOverrides={
                                              "command": [ "python", "batch_processor.py", batchCommand  ],
                                              "environment": [ 
                                                  {"name": "InputBucket", "value": bucketName},
                                                  {"name": "FileName", "value": inputFileName},
                                                  {"name": "Region", "value": region},
                                                  {"name": "DBTableName", "value": db_table_name}
                                              ]
                                          })

              print("Job ID is {}.".format(response['jobId']))
              return response  
  #Code Commit
  CodeCommitRepository:
    Type: AWS::CodeCommit::Repository
    Properties:
      RepositoryName: !Sub ${StackName}-repo
      RepositoryDescription: Respository to maintain code related to the Batch Processing Jobs.
  #Code Build
  CodeBuildProject:
    Type: AWS::CodeBuild::Project
    Properties:
      Name: !Sub ${StackName}-build
      Description: Batch processing application codebuild project.
      ServiceRole: !GetAtt CodeBuildRole.Arn
      Artifacts:
        Type: no_artifacts
      Environment:
        Type: LINUX_CONTAINER
        ComputeType: BUILD_GENERAL1_SMALL
        Image: aws/codebuild/amazonlinux2-x86_64-standard:1.0
        PrivilegedMode: true
        EnvironmentVariables:
            - Name: REPOSITORY_URI
              Type: PLAINTEXT
              Value: 
                Fn::Join:
                - ''
                - - Ref: AWS::AccountId
                  - .dkr.ecr.
                  - Ref: AWS::Region
                  - !Sub '.amazonaws.com/${StackName}-repository:latest'

            - Name: AWS_DEFAULT_REGION
              Type: PLAINTEXT
              Value: 
                Ref: AWS::Region        
      Source:
        BuildSpec: config/buildspec.yml
        Location: 
          Fn::Join:
            - ''
            - - 'https://git-codecommit.'
              - Ref: AWS::Region
              - '.amazonaws.com/v1/repos/'
              - !Sub ${StackName}-repo
        Type: CODECOMMIT 
      SourceVersion: refs/heads/master
      TimeoutInMinutes: 10 
  CodeBuildRole:
    Type: AWS::IAM::Role
    Properties:
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess 
      - arn:aws:iam::aws:policy/AWSCodeCommitFullAccess   
      AssumeRolePolicyDocument:
        Statement:
        - Action: ['sts:AssumeRole']
          Effect: Allow
          Principal:
            Service: [codebuild.amazonaws.com]
        Version: '2012-10-17'
      Path: /
      Policies:
        - PolicyName: CodeBuildAccess
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Action:
                - 'logs:*'
                - 'ec2:CreateNetworkInterface'
                - 'ec2:DescribeNetworkInterfaces'
                - 'ec2:DeleteNetworkInterface'
                - 'ec2:DescribeSubnets'
                - 'ec2:DescribeSecurityGroups'
                - 'ec2:DescribeDhcpOptions'
                - 'ec2:DescribeVpcs'
                - 'ec2:CreateNetworkInterfacePermission'
                Effect: Allow
                Resource: '*'
  # CloudWatchEvents Code build Rold            
  CloudWatchEventsCodeBuildRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub ${StackName}-cw-events-codebuild-role
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          -
            Effect: Allow
            Principal:
              Service:
                - events.amazonaws.com
            Action: sts:AssumeRole
      Policies:
      - PolicyName: aws-events-code-build
        PolicyDocument:
          Version: 2012-10-17
          Statement:
          - Effect: Allow
            Action:
              - 'codebuild:StartBuild'
            Resource: !GetAtt CodeBuildProject.Arn              
  # CloudWatch Event Rule for codecommit build trigger
  CloudWatchEventCodeBuildEventRule: 
    Type: AWS::Events::Rule
    Properties: 
      Description: "This event rule triggers the build on code commit event"
      EventPattern: 
        source: 
          - "aws.codecommit"
        detail-type: 
          - "CodeCommit Repository State Change"
        detail: 
          event: 
            - "referenceCreated"
            - "referenceUpdated"
          referenceType: 
            - "branch"
          referenceName: 
            - "master"                    
      State: "ENABLED"
      Targets: 
        - 
          Arn: {'Fn::GetAtt': [CodeBuildProject, Arn]}
          Id: cloudwatch-codebuild-eventrules  
          RoleArn: !GetAtt CloudWatchEventsCodeBuildRole.Arn
  BatchProcessRepository: 
    Type: AWS::ECR::Repository
    Properties: 
      RepositoryName: !Sub ${StackName}-repository
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



