workshop/setup/existing-studio-domain/CFN-EMR-ExistingStudioNoAuthTemplate.yaml [501:696]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  allowEMRFSAccessForUser1:
    Type: AWS::IAM::Role
    Properties:
      RoleName:
        Fn::Sub: "${AWS::StackName}-allowEMRFSAccessForUser1"
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            AWS:
              Fn::Sub: arn:aws:iam::${AWS::AccountId}:role/${AWS::StackName}-EMRClusterinstanceProfileRole
          Action:
          - sts:AssumeRole
      Path: "/"
      Policies:
      - PolicyName:
          Fn::Sub: "${AWS::StackName}-emrFS-user1"
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Action:
            - s3:ListBucket
            Resource:
            - Fn::Sub: arn:aws:s3:::${S3Bucket}
            Effect: Allow
          - Action:
            - s3:*
            Resource:
            - Fn::Sub: arn:aws:s3:::${S3Bucket}/*
            Effect: Allow
  CopyZips:
    Type: Custom::CopyZips
    Properties:
      ServiceToken:
        Fn::GetAtt: CopyZipsFunction.Arn
      DestBucket:
        Ref: S3Bucket
      SourceBucket:
        Fn::FindInMap:
        - Studio
        - s3params
        - S3Bucket
      Prefix:
        Fn::FindInMap:
        - Studio
        - s3params
        - S3Key
      Objects:
      - Fn::FindInMap:
        - ClusterConfigurations
        - emr
        - BootStrapScriptFile
      - Fn::FindInMap:
        - ClusterConfigurations
        - emr
        - StepScriptFile
  CopyZipsRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service: lambda.amazonaws.com
          Action: sts:AssumeRole
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Path: "/"
      Policies:
      - PolicyName: lambda-copier
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - s3:GetObject
            Resource: "*"
          - Effect: Allow
            Action:
            - s3:PutObject
            - s3:DeleteObject
            Resource:
            - Fn::Sub: arn:aws:s3:::${S3Bucket}/*
  CopyZipsFunction:
    Type: AWS::Lambda::Function
    Properties:
      Description: Copies objects from a source S3 bucket to a destination
      Handler: index.handler
      Runtime: python3.8
      Role:
        Fn::GetAtt: CopyZipsRole.Arn
      Timeout: 900
      Code:
        ZipFile: |
          import json
          import logging
          import threading
          import boto3
          import cfnresponse
          def copy_objects(source_bucket, dest_bucket, prefix, objects):
              s3 = boto3.client('s3')
              for o in objects:
                  key = prefix + o
                  copy_source = {
                      'Bucket': source_bucket,
                      'Key': key
                  }
                  print('copy_source: %s' % copy_source)
                  print('dest_bucket = %s'%dest_bucket)
                  print('key = %s' %key)
                  s3.copy_object(CopySource=copy_source, Bucket=dest_bucket,
                        Key=key)
          def delete_objects(bucket, prefix, objects):
              s3 = boto3.client('s3')
              objects = {'Objects': [{'Key': prefix + o} for o in objects]}
              s3.delete_objects(Bucket=bucket, Delete=objects)
          def timeout(event, context):
              logging.error('Execution is about to time out, sending failure response to CloudFormation')
              cfnresponse.send(event, context, cfnresponse.FAILED, {}, None)
          def handler(event, context):
              # make sure we send a failure to CloudFormation if the function
              # is going to timeout
              timer = threading.Timer((context.get_remaining_time_in_millis()
                        / 1000.00) - 0.5, timeout, args=[event, context])
              timer.start()
              print('Received event: %s' % json.dumps(event))
              status = cfnresponse.SUCCESS
              try:
                  source_bucket = event['ResourceProperties']['SourceBucket']
                  dest_bucket = event['ResourceProperties']['DestBucket']
                  prefix = event['ResourceProperties']['Prefix']
                  objects = event['ResourceProperties']['Objects']
                  if event['RequestType'] == 'Delete':
                      delete_objects(dest_bucket, prefix, objects)
                  else:
                      copy_objects(source_bucket, dest_bucket, prefix, objects)
              except Exception as e:
                  logging.error('Exception: %s' % e, exc_info=True)
                  status = cfnresponse.FAILED
              finally:
                  timer.cancel()
                  cfnresponse.send(event, context, status, {}, None)
  EMRCluster:
    DependsOn: CopyZips
    Type: AWS::EMR::Cluster
    Properties:
      Name:
        Ref: EmrClusterName
      Applications:
      - Name: Spark
      - Name: Hive
      - Name: Livy
      BootstrapActions:
      - Name: Dummy bootstrap action
        ScriptBootstrapAction:
          Args:
          - dummy
          - parameter
          Path:
            Fn::Sub: s3://${S3Bucket}/artifacts/sma-milestone1/installpylibs.sh
      AutoScalingRole: EMR_AutoScaling_DefaultRole
      Configurations:
      - Classification: livy-conf
        ConfigurationProperties:
          livy.server.session.timeout: 2h
      EbsRootVolumeSize: 100
      Instances:
        CoreInstanceGroup:
          EbsConfiguration:
            EbsBlockDeviceConfigs:
            - VolumeSpecification:
                SizeInGB: '320'
                VolumeType: gp2
              VolumesPerInstance: '1'
            EbsOptimized: 'true'
          InstanceCount:
            Ref: CoreInstanceCount
          InstanceType:
            Ref: CoreInstanceType
          Market: ON_DEMAND
          Name: coreNode
        MasterInstanceGroup:
          EbsConfiguration:
            EbsBlockDeviceConfigs:
            - VolumeSpecification:
                SizeInGB: '320'
                VolumeType: gp2
              VolumesPerInstance: '1'
            EbsOptimized: 'true'
          InstanceCount: 1
          InstanceType:
            Ref: MasterInstanceType
          Market: ON_DEMAND
          Name: masterNode
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



workshop/setup/no-existing-studio-domain/CFN-EMR-NoStudioNoAuthTemplate.yaml [501:696]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  allowEMRFSAccessForUser1:
    Type: AWS::IAM::Role
    Properties:
      RoleName:
        Fn::Sub: "${AWS::StackName}-allowEMRFSAccessForUser1"
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            AWS:
              Fn::Sub: arn:aws:iam::${AWS::AccountId}:role/${AWS::StackName}-EMRClusterinstanceProfileRole
          Action:
          - sts:AssumeRole
      Path: "/"
      Policies:
      - PolicyName:
          Fn::Sub: "${AWS::StackName}-emrFS-user1"
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Action:
            - s3:ListBucket
            Resource:
            - Fn::Sub: arn:aws:s3:::${S3Bucket}
            Effect: Allow
          - Action:
            - s3:*
            Resource:
            - Fn::Sub: arn:aws:s3:::${S3Bucket}/*
            Effect: Allow
  CopyZips:
    Type: Custom::CopyZips
    Properties:
      ServiceToken:
        Fn::GetAtt: CopyZipsFunction.Arn
      DestBucket:
        Ref: S3Bucket
      SourceBucket:
        Fn::FindInMap:
        - Studio
        - s3params
        - S3Bucket
      Prefix:
        Fn::FindInMap:
        - Studio
        - s3params
        - S3Key
      Objects:
      - Fn::FindInMap:
        - ClusterConfigurations
        - emr
        - BootStrapScriptFile
      - Fn::FindInMap:
        - ClusterConfigurations
        - emr
        - StepScriptFile
  CopyZipsRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service: lambda.amazonaws.com
          Action: sts:AssumeRole
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Path: "/"
      Policies:
      - PolicyName: lambda-copier
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - s3:GetObject
            Resource: "*"
          - Effect: Allow
            Action:
            - s3:PutObject
            - s3:DeleteObject
            Resource:
            - Fn::Sub: arn:aws:s3:::${S3Bucket}/*
  CopyZipsFunction:
    Type: AWS::Lambda::Function
    Properties:
      Description: Copies objects from a source S3 bucket to a destination
      Handler: index.handler
      Runtime: python3.8
      Role:
        Fn::GetAtt: CopyZipsRole.Arn
      Timeout: 900
      Code:
        ZipFile: |
          import json
          import logging
          import threading
          import boto3
          import cfnresponse
          def copy_objects(source_bucket, dest_bucket, prefix, objects):
              s3 = boto3.client('s3')
              for o in objects:
                  key = prefix + o
                  copy_source = {
                      'Bucket': source_bucket,
                      'Key': key
                  }
                  print('copy_source: %s' % copy_source)
                  print('dest_bucket = %s'%dest_bucket)
                  print('key = %s' %key)
                  s3.copy_object(CopySource=copy_source, Bucket=dest_bucket,
                        Key=key)
          def delete_objects(bucket, prefix, objects):
              s3 = boto3.client('s3')
              objects = {'Objects': [{'Key': prefix + o} for o in objects]}
              s3.delete_objects(Bucket=bucket, Delete=objects)
          def timeout(event, context):
              logging.error('Execution is about to time out, sending failure response to CloudFormation')
              cfnresponse.send(event, context, cfnresponse.FAILED, {}, None)
          def handler(event, context):
              # make sure we send a failure to CloudFormation if the function
              # is going to timeout
              timer = threading.Timer((context.get_remaining_time_in_millis()
                        / 1000.00) - 0.5, timeout, args=[event, context])
              timer.start()
              print('Received event: %s' % json.dumps(event))
              status = cfnresponse.SUCCESS
              try:
                  source_bucket = event['ResourceProperties']['SourceBucket']
                  dest_bucket = event['ResourceProperties']['DestBucket']
                  prefix = event['ResourceProperties']['Prefix']
                  objects = event['ResourceProperties']['Objects']
                  if event['RequestType'] == 'Delete':
                      delete_objects(dest_bucket, prefix, objects)
                  else:
                      copy_objects(source_bucket, dest_bucket, prefix, objects)
              except Exception as e:
                  logging.error('Exception: %s' % e, exc_info=True)
                  status = cfnresponse.FAILED
              finally:
                  timer.cancel()
                  cfnresponse.send(event, context, status, {}, None)
  EMRCluster:
    DependsOn: CopyZips
    Type: AWS::EMR::Cluster
    Properties:
      Name:
        Ref: EmrClusterName
      Applications:
      - Name: Spark
      - Name: Hive
      - Name: Livy
      BootstrapActions:
      - Name: Dummy bootstrap action
        ScriptBootstrapAction:
          Args:
          - dummy
          - parameter
          Path:
            Fn::Sub: s3://${S3Bucket}/artifacts/sma-milestone1/installpylibs.sh
      AutoScalingRole: EMR_AutoScaling_DefaultRole
      Configurations:
      - Classification: livy-conf
        ConfigurationProperties:
          livy.server.session.timeout: 2h
      EbsRootVolumeSize: 100
      Instances:
        CoreInstanceGroup:
          EbsConfiguration:
            EbsBlockDeviceConfigs:
            - VolumeSpecification:
                SizeInGB: '320'
                VolumeType: gp2
              VolumesPerInstance: '1'
            EbsOptimized: 'true'
          InstanceCount:
            Ref: CoreInstanceCount
          InstanceType:
            Ref: CoreInstanceType
          Market: ON_DEMAND
          Name: coreNode
        MasterInstanceGroup:
          EbsConfiguration:
            EbsBlockDeviceConfigs:
            - VolumeSpecification:
                SizeInGB: '320'
                VolumeType: gp2
              VolumesPerInstance: '1'
            EbsOptimized: 'true'
          InstanceCount: 1
          InstanceType:
            Ref: MasterInstanceType
          Market: ON_DEMAND
          Name: masterNode
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



