emr_templates/single-account.yaml [184:490]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  emrMasterIngressServiceSg:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: masterSecurityGroup
      IpProtocol: tcp
      FromPort: 8443
      ToPort: 8443
      SourceSecurityGroupId:
        Ref: emrServiceSecurityGroup
  emrServiceIngressMasterSg:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: emrServiceSecurityGroup
      IpProtocol: tcp
      FromPort: 9443
      ToPort: 9443
      SourceSecurityGroupId:
        Ref: masterSecurityGroup
  emrServiceEgressMaster:
    Type: AWS::EC2::SecurityGroupEgress
    Properties:
      GroupId:
        Ref: emrServiceSecurityGroup
      IpProtocol: tcp
      FromPort: 8443
      ToPort: 8443
      DestinationSecurityGroupId:
        Ref: masterSecurityGroup
  emrServiceEgressSlave:
    Type: AWS::EC2::SecurityGroupEgress
    Properties:
      GroupId:
        Ref: emrServiceSecurityGroup
      IpProtocol: tcp
      FromPort: 8443
      ToPort: 8443
      DestinationSecurityGroupId:
        Ref: slaveSecurityGroup
  emrSlaveIngressSelfICMP:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: icmp
      FromPort: -1
      ToPort: -1
      SourceSecurityGroupId:
        Ref: slaveSecurityGroup
  emrSlaveIngressMasterICMP:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: icmp
      FromPort: -1
      ToPort: -1
      SourceSecurityGroupId:
        Ref: masterSecurityGroup
  emrSlaveIngressSelfAllTcp:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: tcp
      FromPort: 0
      ToPort: 65535
      SourceSecurityGroupId:
        Ref: slaveSecurityGroup
  emrSlaveIngressMasterAllTcp:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: tcp
      FromPort: 0
      ToPort: 65535
      SourceSecurityGroupId:
        Ref: masterSecurityGroup
  emrSlaveIngressSelfAllUdp:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: udp
      FromPort: 0
      ToPort: 65535
      SourceSecurityGroupId:
        Ref: slaveSecurityGroup
  emrSlaveIngressMasterAllUdp:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: udp
      FromPort: 0
      ToPort: 65535
      SourceSecurityGroupId:
        Ref: masterSecurityGroup
  emrSlaveIngressServiceSg:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: tcp
      FromPort: 8443
      ToPort: 8443
      SourceSecurityGroupId:
        Ref: emrServiceSecurityGroup
  EMRClusterServiceRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Effect: Allow
          Principal:
            Service:
            - elasticmapreduce.amazonaws.com
        Version: '2012-10-17'
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole
      Path: "/"
    Type: AWS::IAM::Role
  EMRClusterinstanceProfile:
    Properties:
      Path: "/"
      Roles:
      - Ref: EMRClusterinstanceProfileRole
    Type: AWS::IAM::InstanceProfile
  EMRClusterinstanceProfileRole:
    Properties:
      RoleName:
        Fn::Sub: "${AWS::StackName}-EMRClusterinstanceProfileRole"
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
        Version: '2012-10-17'
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role
      Path: "/"
    Type: AWS::IAM::Role
  CleanUpBucketonDelete:
    DependsOn: CleanUpBucketonDeleteLambda
    Type: Custom::emptybucket
    Properties:
      ServiceToken:
        Fn::GetAtt:
        - CleanUpBucketonDeleteLambda
        - Arn
      inputBucketName:
        Ref: S3Bucket
  CleanUpBucketonDeleteLambda:
    DependsOn:
    - S3Bucket
    - CleanUpBucketonDeleteLambdaRole
    Type: AWS::Lambda::Function
    Properties:
      Description: Empty bucket on delete
      Handler: index.lambda_handler
      Role:
        Fn::GetAtt:
        - CleanUpBucketonDeleteLambdaRole
        - Arn
      Runtime: python3.7
      Timeout: 60
      Code:
        ZipFile:
          Fn::Join:
          - "\n"
          - - import json
            - import boto3
            - import urllib3
            - ''
            - 'def empty_bucket(bucket_name):'
            - '    print("Attempting to empty the bucket {0}".format(bucket_name))'
            - "    s3_client = boto3.client('s3')"
            - "    s3 = boto3.resource('s3')"
            - ''
            - "    try:"
            - "        bucket = s3.Bucket(bucket_name).load()"
            - "    except ClientError:"
            - '        print("Bucket {0} does not exist".format(bucket_name))'
            - "        return"
            - "    # Confirm if versioning is enabled"
            - "    version_status = s3_client.get_bucket_versioning(Bucket=bucket_name)"
            - "    status = version_status.get('Status','')"
            - "    if status == 'Enabled':"
            - "        version_status = s3_client.put_bucket_versioning(Bucket=bucket_name,"
            - "                                                   VersioningConfiguration={'Status':
              'Suspended'})"
            - "    version_paginator = s3_client.get_paginator('list_object_versions')"
            - "    version_iterator = version_paginator.paginate("
            - "        Bucket=bucket_name"
            - "    )"
            - ''
            - "    for page in version_iterator:"
            - "        print(page)"
            - "        if 'DeleteMarkers' in page:"
            - "            delete_markers = page['DeleteMarkers']"
            - "            if delete_markers is not None:"
            - "                for delete_marker in delete_markers:"
            - "                    key = delete_marker['Key']"
            - "                    versionId = delete_marker['VersionId']"
            - "                    s3_client.delete_object(Bucket=bucket_name, Key=key,
              VersionId=versionId)"
            - "        if 'Versions' in page and page['Versions'] is not None:"
            - "            versions = page['Versions']"
            - "            for version in versions:"
            - "                print(version)"
            - "                key = version['Key']"
            - "                versionId = version['VersionId']"
            - "                s3_client.delete_object(Bucket=bucket_name, Key=key,
              VersionId=versionId)"
            - "    object_paginator = s3_client.get_paginator('list_objects_v2')"
            - "    object_iterator = object_paginator.paginate("
            - "        Bucket=bucket_name"
            - "    )"
            - "    for page in object_iterator:"
            - "        if 'Contents' in page:"
            - "            for content in page['Contents']:"
            - "                key = content['Key']"
            - "                s3_client.delete_object(Bucket=bucket_name, Key=content['Key'])"
            - '    print("Successfully emptied the bucket {0}".format(bucket_name))'
            - ''
            - ''
            - ''
            - 'def lambda_handler(event, context):'
            - "    try:"
            - "        bucket = event['ResourceProperties']['inputBucketName']"
            - "        if event['RequestType'] == 'Delete':"
            - "            empty_bucket(bucket)"
            - '        sendResponse(event, context, "SUCCESS")'
            - "    except Exception as e:"
            - "        print(e)"
            - '        sendResponse(event, context, "FAILED")'
            - ''
            - 'def sendResponse(event, context, status):'
            - "    http = urllib3.PoolManager()"
            - "    response_body = {'Status': status,"
            - "                     'Reason': 'Log stream name: ' + context.log_stream_name,"
            - "                     'PhysicalResourceId': context.log_stream_name,"
            - "                     'StackId': event['StackId'],"
            - "                     'RequestId': event['RequestId'],"
            - "                     'LogicalResourceId': event['LogicalResourceId'],"
            - '                     ''Data'': json.loads("{}")}'
            - "    http.request('PUT', event['ResponseURL'], body=json.dumps(response_body))"
  CleanUpBucketonDeleteLambdaRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: "/"
      Policies:
      - PolicyName:
          Fn::Sub: CleanUpBucketonDeleteLambdaPolicy-${AWS::StackName}
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - s3:*
            Resource:
            - Fn::GetAtt:
              - S3Bucket
              - Arn
            - Fn::Join:
              - ''
              - - Fn::GetAtt:
                  - S3Bucket
                  - Arn
                - "/"
            - Fn::Join:
              - ''
              - - Fn::GetAtt:
                  - S3Bucket
                  - Arn
                - "/*"
          - Effect: Deny
            Action:
            - s3:DeleteBucket
            Resource: "*"
          - Effect: Allow
            Action:
            - logs:*
            Resource: "*"
  emrMasterIngressKDCSG:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: masterSecurityGroup
      IpProtocol: tcp
      FromPort: 88
      ToPort: 88
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



workshop/setup/existing-studio-domain/CFN-EMR-ExistingStudioNoAuthTemplate.yaml [175:481]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  emrMasterIngressServiceSg:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: masterSecurityGroup
      IpProtocol: tcp
      FromPort: 8443
      ToPort: 8443
      SourceSecurityGroupId:
        Ref: emrServiceSecurityGroup
  emrServiceIngressMasterSg:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: emrServiceSecurityGroup
      IpProtocol: tcp
      FromPort: 9443
      ToPort: 9443
      SourceSecurityGroupId:
        Ref: masterSecurityGroup
  emrServiceEgressMaster:
    Type: AWS::EC2::SecurityGroupEgress
    Properties:
      GroupId:
        Ref: emrServiceSecurityGroup
      IpProtocol: tcp
      FromPort: 8443
      ToPort: 8443
      DestinationSecurityGroupId:
        Ref: masterSecurityGroup
  emrServiceEgressSlave:
    Type: AWS::EC2::SecurityGroupEgress
    Properties:
      GroupId:
        Ref: emrServiceSecurityGroup
      IpProtocol: tcp
      FromPort: 8443
      ToPort: 8443
      DestinationSecurityGroupId:
        Ref: slaveSecurityGroup
  emrSlaveIngressSelfICMP:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: icmp
      FromPort: -1
      ToPort: -1
      SourceSecurityGroupId:
        Ref: slaveSecurityGroup
  emrSlaveIngressMasterICMP:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: icmp
      FromPort: -1
      ToPort: -1
      SourceSecurityGroupId:
        Ref: masterSecurityGroup
  emrSlaveIngressSelfAllTcp:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: tcp
      FromPort: 0
      ToPort: 65535
      SourceSecurityGroupId:
        Ref: slaveSecurityGroup
  emrSlaveIngressMasterAllTcp:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: tcp
      FromPort: 0
      ToPort: 65535
      SourceSecurityGroupId:
        Ref: masterSecurityGroup
  emrSlaveIngressSelfAllUdp:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: udp
      FromPort: 0
      ToPort: 65535
      SourceSecurityGroupId:
        Ref: slaveSecurityGroup
  emrSlaveIngressMasterAllUdp:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: udp
      FromPort: 0
      ToPort: 65535
      SourceSecurityGroupId:
        Ref: masterSecurityGroup
  emrSlaveIngressServiceSg:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: slaveSecurityGroup
      IpProtocol: tcp
      FromPort: 8443
      ToPort: 8443
      SourceSecurityGroupId:
        Ref: emrServiceSecurityGroup
  EMRClusterServiceRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Effect: Allow
          Principal:
            Service:
            - elasticmapreduce.amazonaws.com
        Version: '2012-10-17'
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole
      Path: "/"
    Type: AWS::IAM::Role
  EMRClusterinstanceProfile:
    Properties:
      Path: "/"
      Roles:
      - Ref: EMRClusterinstanceProfileRole
    Type: AWS::IAM::InstanceProfile
  EMRClusterinstanceProfileRole:
    Properties:
      RoleName:
        Fn::Sub: "${AWS::StackName}-EMRClusterinstanceProfileRole"
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
        Version: '2012-10-17'
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role
      Path: "/"
    Type: AWS::IAM::Role
  CleanUpBucketonDelete:
    DependsOn: CleanUpBucketonDeleteLambda
    Type: Custom::emptybucket
    Properties:
      ServiceToken:
        Fn::GetAtt:
        - CleanUpBucketonDeleteLambda
        - Arn
      inputBucketName:
        Ref: S3Bucket
  CleanUpBucketonDeleteLambda:
    DependsOn:
    - S3Bucket
    - CleanUpBucketonDeleteLambdaRole
    Type: AWS::Lambda::Function
    Properties:
      Description: Empty bucket on delete
      Handler: index.lambda_handler
      Role:
        Fn::GetAtt:
        - CleanUpBucketonDeleteLambdaRole
        - Arn
      Runtime: python3.7
      Timeout: 60
      Code:
        ZipFile:
          Fn::Join:
          - "\n"
          - - import json
            - import boto3
            - import urllib3
            - ''
            - 'def empty_bucket(bucket_name):'
            - '    print("Attempting to empty the bucket {0}".format(bucket_name))'
            - "    s3_client = boto3.client('s3')"
            - "    s3 = boto3.resource('s3')"
            - ''
            - "    try:"
            - "        bucket = s3.Bucket(bucket_name).load()"
            - "    except ClientError:"
            - '        print("Bucket {0} does not exist".format(bucket_name))'
            - "        return"
            - "    # Confirm if versioning is enabled"
            - "    version_status = s3_client.get_bucket_versioning(Bucket=bucket_name)"
            - "    status = version_status.get('Status','')"
            - "    if status == 'Enabled':"
            - "        version_status = s3_client.put_bucket_versioning(Bucket=bucket_name,"
            - "                                                   VersioningConfiguration={'Status':
              'Suspended'})"
            - "    version_paginator = s3_client.get_paginator('list_object_versions')"
            - "    version_iterator = version_paginator.paginate("
            - "        Bucket=bucket_name"
            - "    )"
            - ''
            - "    for page in version_iterator:"
            - "        print(page)"
            - "        if 'DeleteMarkers' in page:"
            - "            delete_markers = page['DeleteMarkers']"
            - "            if delete_markers is not None:"
            - "                for delete_marker in delete_markers:"
            - "                    key = delete_marker['Key']"
            - "                    versionId = delete_marker['VersionId']"
            - "                    s3_client.delete_object(Bucket=bucket_name, Key=key,
              VersionId=versionId)"
            - "        if 'Versions' in page and page['Versions'] is not None:"
            - "            versions = page['Versions']"
            - "            for version in versions:"
            - "                print(version)"
            - "                key = version['Key']"
            - "                versionId = version['VersionId']"
            - "                s3_client.delete_object(Bucket=bucket_name, Key=key,
              VersionId=versionId)"
            - "    object_paginator = s3_client.get_paginator('list_objects_v2')"
            - "    object_iterator = object_paginator.paginate("
            - "        Bucket=bucket_name"
            - "    )"
            - "    for page in object_iterator:"
            - "        if 'Contents' in page:"
            - "            for content in page['Contents']:"
            - "                key = content['Key']"
            - "                s3_client.delete_object(Bucket=bucket_name, Key=content['Key'])"
            - '    print("Successfully emptied the bucket {0}".format(bucket_name))'
            - ''
            - ''
            - ''
            - 'def lambda_handler(event, context):'
            - "    try:"
            - "        bucket = event['ResourceProperties']['inputBucketName']"
            - "        if event['RequestType'] == 'Delete':"
            - "            empty_bucket(bucket)"
            - '        sendResponse(event, context, "SUCCESS")'
            - "    except Exception as e:"
            - "        print(e)"
            - '        sendResponse(event, context, "FAILED")'
            - ''
            - 'def sendResponse(event, context, status):'
            - "    http = urllib3.PoolManager()"
            - "    response_body = {'Status': status,"
            - "                     'Reason': 'Log stream name: ' + context.log_stream_name,"
            - "                     'PhysicalResourceId': context.log_stream_name,"
            - "                     'StackId': event['StackId'],"
            - "                     'RequestId': event['RequestId'],"
            - "                     'LogicalResourceId': event['LogicalResourceId'],"
            - '                     ''Data'': json.loads("{}")}'
            - "    http.request('PUT', event['ResponseURL'], body=json.dumps(response_body))"
  CleanUpBucketonDeleteLambdaRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: "/"
      Policies:
      - PolicyName:
          Fn::Sub: CleanUpBucketonDeleteLambdaPolicy-${AWS::StackName}
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - s3:*
            Resource:
            - Fn::GetAtt:
              - S3Bucket
              - Arn
            - Fn::Join:
              - ''
              - - Fn::GetAtt:
                  - S3Bucket
                  - Arn
                - "/"
            - Fn::Join:
              - ''
              - - Fn::GetAtt:
                  - S3Bucket
                  - Arn
                - "/*"
          - Effect: Deny
            Action:
            - s3:DeleteBucket
            Resource: "*"
          - Effect: Allow
            Action:
            - logs:*
            Resource: "*"
  emrMasterIngressKDCSG:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId:
        Ref: masterSecurityGroup
      IpProtocol: tcp
      FromPort: 88
      ToPort: 88
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



