static/Reliability/200_Backup_Restore_Failback_Analytics/Code/cfn/dr-infra.yaml [560:796]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  LambdaFunctionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
              - lambda.amazonaws.com
          Action:
            - sts:AssumeRole
      Path: "/"
      Policies:
      - PolicyName: LambdaFunctionPolicy
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
              - logs:CreateLogGroup
              - logs:CreateLogStream
              - logs:PutLogEvents
              - dynamodb:*
              - kinesis:*
            Resource: '*'

  ProcessedLambdaFunction:
    Type: AWS::Lambda::Function
    Properties:
      Runtime: python3.6
      Timeout: 5
      Handler: index.handler
      Role: !GetAtt LambdaFunctionRole.Arn
      Code:
        ZipFile:
          !Sub
            - |-
              #!/usr/bin/env python3

              import base64
              import json
              import boto3

              print('Loading function')


              def handler(event, context):
                #print("Received event: " + json.dumps(event, indent=2))
                dynamodb = boto3.resource('dynamodb')
                table = dynamodb.Table('processed_tweets')
                for record in event['Records']:
                  # Kinesis data is base64 encoded so decode here
                  id=(record['kinesis']['sequenceNumber'])
                  payload = base64.b64decode(record['kinesis']['data']).decode('utf-8')
        
                  table.put_item(Item= {'id': id,'payload':  payload})

                  print("Decoded payload: " + payload)
                return 'Successfully processed {} records.'.format(len(event['Records']))
            
            -
              lambda_function_role_arn: !Ref LambdaFunctionRole

  InboundStreamLambdaFunctionEventSourceMapping:
    Type: AWS::Lambda::EventSourceMapping
    Properties: 
      BatchSize: 100 
      Enabled: true
      EventSourceArn: !GetAtt ProcessedStream.Arn
      FunctionName: !GetAtt ProcessedLambdaFunction.Arn
      MaximumBatchingWindowInSeconds: 0
      StartingPosition: LATEST 


  KinesisAnalyticsApp:
    Type: AWS::KinesisAnalytics::Application
    Properties:
      ApplicationName: !Sub '${AWS::StackName}-KinesisAnalyticsApplication'
      ApplicationDescription: Kineis Analytics Solution Accelerator
      ApplicationCode: !Sub |
            -- Approximate distinct count  - Counts the number of distinct items in a stream using HyperLogLog.
            -- Returns the approximate number of distinct items in a specified column over a tumbling window.
            -- Note that when there are less or equal to 10,000 items in the window, the function returns exact count.
            CREATE OR REPLACE STREAM DESTINATION_SQL_STREAM (NUMBER_OF_DISTINCT_ITEMS BIGINT);
            CREATE OR REPLACE PUMP "STREAM_PUMP" AS INSERT INTO "DESTINATION_SQL_STREAM"
            SELECT STREAM NUMBER_OF_DISTINCT_ITEMS FROM TABLE(COUNT_DISTINCT_ITEMS_TUMBLING(
              CURSOR(SELECT STREAM * FROM "SOURCE_SQL_STREAM_001"),
              'favorite_count', -- name of column in single quotes
              60 -- tumbling window size in seconds
              )
            );


      Inputs:
        - NamePrefix: SOURCE_SQL_STREAM
          InputSchema:
            RecordColumns:
              - Name: type
                SqlType: VARCHAR(8)
                Mapping: $.type
              - Name: coordinates
                SqlType: VARCHAR(32)
                Mapping: $.coordinates
              - Name: retweeted
                SqlType: BOOLEAN
                Mapping: $.retweeted
              - Name: source
                SqlType: VARCHAR(128)
                Mapping: $.source
              - Name: hashtags
                SqlType: VARCHAR(1024)
                Mapping: $.hashtags
              - Name: urls
                SqlType: VARCHAR(1024)
                Mapping: $.urls
              - Name: agent
                SqlType: VARCHAR(128)
                Mapping: $.agent
              - Name: reply_count
                SqlType: INTEGER
                Mapping: $.reply_count
              - Name: favorite_count
                SqlType: INTEGER
                Mapping: $.favorite_count
              - Name: type0
                SqlType: VARCHAR(8)
                Mapping: $.type0
              - Name: coordinates0
                SqlType: VARCHAR(32)
                Mapping: $.coordinates0
              - Name: id_str
                SqlType: VARCHAR(32)
                Mapping: $.id_str
              - Name: timestamp_ms
                SqlType: BIGINT
                Mapping: $.timestamp_ms
              - Name: truncated
                SqlType: BOOLEAN
                Mapping: $.truncated
              - Name: retweet_count
                SqlType: INTEGER
                Mapping: $.retweet_count
              - Name: id
                SqlType: INTEGER
                Mapping: $.id  
              - Name: possibly_sensitive
                SqlType: BOOLEAN
                Mapping: $.possibly_sensitive  
              - Name: filter_level
                SqlType: VARCHAR(32)
                Mapping: $.filter_level  
              - Name: quote_count
                SqlType: INTEGER
                Mapping: $.quote_count
              - Name: lang
                SqlType: VARCHAR(4)
                Mapping: $.lang  
              - Name: favorited
                SqlType: BOOLEAN
                Mapping: $.favorited  
              - Name: is_quote_status
                SqlType: BOOLEAN
                Mapping: $.is_quote_status 
              - Name: created_at
                SqlType: VARCHAR(16)
                Mapping: $.created_at 
              - Name: in_reply_to_screen_name
                SqlType: VARCHAR(16)
                Mapping: $.in_reply_to_screen_name 
              - Name: in_reply_to_user_id_str
                SqlType: VARCHAR(16)
                Mapping: $.in_reply_to_user_id_str 
              - Name: text
                SqlType: VARCHAR(64)
                Mapping: $.text 
              - Name: id0
                SqlType: INTEGER
                Mapping: $.id0 
              - Name: url
                SqlType: VARCHAR(32)
                Mapping: $.url 
              - Name: place_type
                SqlType: VARCHAR(16)
                Mapping: $.place_type 
              - Name: name
                SqlType: VARCHAR(32)
                Mapping: $.name 
              - Name: full_name
                SqlType: VARCHAR(32)
                Mapping: $.full_name 
              - Name: country_code
                SqlType: VARCHAR(4)
                Mapping: $.country_code 
              - Name: country
                SqlType: VARCHAR(64)
                Mapping: $.country 
              - Name: type1
                SqlType: VARCHAR(8)
                Mapping: $.type1 
              - Name: coordinates1
                SqlType: VARCHAR(1024)
                Mapping: $.coordinates1 
              - Name: id1
                SqlType: INTEGER
                Mapping: $.id1 
              - Name: id_str0
                SqlType: INTEGER
                Mapping: $.id1 
            RecordFormat:
              RecordFormatType: JSON
              MappingParameters:
                JSONMappingParameters:
                  RecordRowPath: $
          KinesisStreamsInput:
            ResourceARN: !GetAtt RawStream.Arn
            RoleARN: !GetAtt KinesisAnalyticsRole.Arn

  KinesisAnalyticsRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service: kinesisanalytics.amazonaws.com
            Action: "sts:AssumeRole"
      Path: "/"
      Policies:
        - PolicyName: Open
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action: "*"
                Resource: "*"
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



static/Reliability/200_Backup_Restore_Failback_Analytics/Code/cfn/workload.yaml [675:911]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  LambdaFunctionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
              - lambda.amazonaws.com
          Action:
            - sts:AssumeRole
      Path: "/"
      Policies:
      - PolicyName: LambdaFunctionPolicy
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
              - logs:CreateLogGroup
              - logs:CreateLogStream
              - logs:PutLogEvents
              - dynamodb:*
              - kinesis:*
            Resource: '*'

  ProcessedLambdaFunction:
    Type: AWS::Lambda::Function
    Properties:
      Runtime: python3.6
      Timeout: 5
      Handler: index.handler
      Role: !GetAtt LambdaFunctionRole.Arn
      Code:
        ZipFile:
          !Sub
            - |-
              #!/usr/bin/env python3

              import base64
              import json
              import boto3

              print('Loading function')


              def handler(event, context):
                #print("Received event: " + json.dumps(event, indent=2))
                dynamodb = boto3.resource('dynamodb')
                table = dynamodb.Table('processed_tweets')
                for record in event['Records']:
                  # Kinesis data is base64 encoded so decode here
                  id=(record['kinesis']['sequenceNumber'])
                  payload = base64.b64decode(record['kinesis']['data']).decode('utf-8')
        
                  table.put_item(Item= {'id': id,'payload':  payload})

                  print("Decoded payload: " + payload)
                return 'Successfully processed {} records.'.format(len(event['Records']))
            
            -
              lambda_function_role_arn: !Ref LambdaFunctionRole

  InboundStreamLambdaFunctionEventSourceMapping:
    Type: AWS::Lambda::EventSourceMapping
    Properties: 
      BatchSize: 100 
      Enabled: true
      EventSourceArn: !GetAtt ProcessedStream.Arn
      FunctionName: !GetAtt ProcessedLambdaFunction.Arn
      MaximumBatchingWindowInSeconds: 0
      StartingPosition: LATEST 


  KinesisAnalyticsApp:
    Type: AWS::KinesisAnalytics::Application
    Properties:
      ApplicationName: !Sub '${AWS::StackName}-KinesisAnalyticsApplication'
      ApplicationDescription: Kineis Analytics Solution Accelerator
      ApplicationCode: !Sub |
            -- Approximate distinct count  - Counts the number of distinct items in a stream using HyperLogLog.
            -- Returns the approximate number of distinct items in a specified column over a tumbling window.
            -- Note that when there are less or equal to 10,000 items in the window, the function returns exact count.
            CREATE OR REPLACE STREAM DESTINATION_SQL_STREAM (NUMBER_OF_DISTINCT_ITEMS BIGINT);
            CREATE OR REPLACE PUMP "STREAM_PUMP" AS INSERT INTO "DESTINATION_SQL_STREAM"
            SELECT STREAM NUMBER_OF_DISTINCT_ITEMS FROM TABLE(COUNT_DISTINCT_ITEMS_TUMBLING(
              CURSOR(SELECT STREAM * FROM "SOURCE_SQL_STREAM_001"),
              'favorite_count', -- name of column in single quotes
              60 -- tumbling window size in seconds
              )
            );


      Inputs:
        - NamePrefix: SOURCE_SQL_STREAM
          InputSchema:
            RecordColumns:
              - Name: type
                SqlType: VARCHAR(8)
                Mapping: $.type
              - Name: coordinates
                SqlType: VARCHAR(32)
                Mapping: $.coordinates
              - Name: retweeted
                SqlType: BOOLEAN
                Mapping: $.retweeted
              - Name: source
                SqlType: VARCHAR(128)
                Mapping: $.source
              - Name: hashtags
                SqlType: VARCHAR(1024)
                Mapping: $.hashtags
              - Name: urls
                SqlType: VARCHAR(1024)
                Mapping: $.urls
              - Name: agent
                SqlType: VARCHAR(128)
                Mapping: $.agent
              - Name: reply_count
                SqlType: INTEGER
                Mapping: $.reply_count
              - Name: favorite_count
                SqlType: INTEGER
                Mapping: $.favorite_count
              - Name: type0
                SqlType: VARCHAR(8)
                Mapping: $.type0
              - Name: coordinates0
                SqlType: VARCHAR(32)
                Mapping: $.coordinates0
              - Name: id_str
                SqlType: VARCHAR(32)
                Mapping: $.id_str
              - Name: timestamp_ms
                SqlType: BIGINT
                Mapping: $.timestamp_ms
              - Name: truncated
                SqlType: BOOLEAN
                Mapping: $.truncated
              - Name: retweet_count
                SqlType: INTEGER
                Mapping: $.retweet_count
              - Name: id
                SqlType: INTEGER
                Mapping: $.id  
              - Name: possibly_sensitive
                SqlType: BOOLEAN
                Mapping: $.possibly_sensitive  
              - Name: filter_level
                SqlType: VARCHAR(32)
                Mapping: $.filter_level  
              - Name: quote_count
                SqlType: INTEGER
                Mapping: $.quote_count
              - Name: lang
                SqlType: VARCHAR(4)
                Mapping: $.lang  
              - Name: favorited
                SqlType: BOOLEAN
                Mapping: $.favorited  
              - Name: is_quote_status
                SqlType: BOOLEAN
                Mapping: $.is_quote_status 
              - Name: created_at
                SqlType: VARCHAR(16)
                Mapping: $.created_at 
              - Name: in_reply_to_screen_name
                SqlType: VARCHAR(16)
                Mapping: $.in_reply_to_screen_name 
              - Name: in_reply_to_user_id_str
                SqlType: VARCHAR(16)
                Mapping: $.in_reply_to_user_id_str 
              - Name: text
                SqlType: VARCHAR(64)
                Mapping: $.text 
              - Name: id0
                SqlType: INTEGER
                Mapping: $.id0 
              - Name: url
                SqlType: VARCHAR(32)
                Mapping: $.url 
              - Name: place_type
                SqlType: VARCHAR(16)
                Mapping: $.place_type 
              - Name: name
                SqlType: VARCHAR(32)
                Mapping: $.name 
              - Name: full_name
                SqlType: VARCHAR(32)
                Mapping: $.full_name 
              - Name: country_code
                SqlType: VARCHAR(4)
                Mapping: $.country_code 
              - Name: country
                SqlType: VARCHAR(64)
                Mapping: $.country 
              - Name: type1
                SqlType: VARCHAR(8)
                Mapping: $.type1 
              - Name: coordinates1
                SqlType: VARCHAR(1024)
                Mapping: $.coordinates1 
              - Name: id1
                SqlType: INTEGER
                Mapping: $.id1 
              - Name: id_str0
                SqlType: INTEGER
                Mapping: $.id1 
            RecordFormat:
              RecordFormatType: JSON
              MappingParameters:
                JSONMappingParameters:
                  RecordRowPath: $
          KinesisStreamsInput:
            ResourceARN: !GetAtt RawStream.Arn
            RoleARN: !GetAtt KinesisAnalyticsRole.Arn

  KinesisAnalyticsRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service: kinesisanalytics.amazonaws.com
            Action: "sts:AssumeRole"
      Path: "/"
      Policies:
        - PolicyName: Open
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action: "*"
                Resource: "*"
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



