module-2/2-kinesis-analytics-module-completed.yaml [1:691]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
AWSTemplateFormatVersion: '2010-09-09'
Description: "Module 2 - adding Kinesis Analytics, Processing Lambda, and DyanmoDB persistence"

Parameters:
  BeaconSourceCIDR:
    Type: String
    Description: "IP address CIDR block that send events to the Beacon Servers. This must allow web site user browsers to access the beacon servers."
    Default: "0.0.0.0/0"
    MinLength: 9
    MaxLength: 18
    AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
    ConstraintDescription: "must be a valid IP CIDR range of the form x.x.x.x/x."
  CorsOrig:
    Type: String
    Description: "This value will be returned by the API in the Access-Control-Allow-Origin header. A star (*) value will support any origin. We recommend specifying a specific origin (e.g. http://example.domain) to restrict cross-site access to your API."
    Default: "*"
  VpcCidr:
    Type: String
    Description: "CIDR block for new VPC for analytics beacon servers."
    Default: 10.0.0.0/23
  Subnet0Cidr:
    Type: String
    Description: "CIDR block for new VPC public subnet #1."
    Default: 10.0.0.0/24
  Subnet1Cidr:
    Type: String
    Description: "CIDR block for new VPC public subnet #2."
    Default: 10.0.1.0/24
  AZ0:
    Type: String
    Description: "Optional: Availability Zone number for public subnet #1."
    Default: 0
    AllowedValues:
      - 0
      - 1
      - 2
      - 3
      - 4
      - 5
  AZ1:
    Type: String
    Description: "Optional: Availability Zone number for public subnet #2."
    Default: 1
    AllowedValues:
      - 0
      - 1
      - 2
      - 3
      - 4
      - 5
  NodeSize:
    Type: String
    Description: "Number of requests per minute that each beacon node will support (50K = t2.medium, 100K = m5.large)."
    Default: 50K
    AllowedValues:
      - 50K
      - 100K
  AutoScalingMinSize:
    Type: Number
    Description: "Minimum number of beacon webservers (must be at least 2 for high availability)."
    Default: 2
    MinValue: 1
  AutoScalingMaxSize:
    Type: Number
    Description: "Maximum number of beacon webservers."
    Default: 6
    MinValue: 1
  CWDashboard:
    Type: String
    Description: "Create a CloudWatch dashboard for beacon server operational metrics?"
    Default: Yes
    AllowedValues:
      - Yes
      - No
  LatestAmiId:
    Type: 'AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>'
    Default: '/aws/service/ami-amazon-linux-latest/amzn-ami-hvm-x86_64-gp2'

Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: "Beacon Web Server Configuration"
        Parameters:
          - CWDashboard
          - BeaconSourceCIDR
          - CorsOrig
          - NodeSize
          - AutoScalingMinSize
          - AutoScalingMaxSize
      - Label:
          default: "Optional: VPC Configuration"
        Parameters:
          - VpcCidr
          - Subnet0Cidr
          - Subnet1Cidr
          - AZ0
          - AZ1
    ParameterLabels:
      CWDashboard:
        default: CloudWatch Dashboard
      BeaconSourceCIDR:
        default: Enable beacon events from
      CorsOrig:
        default: CORS Origin
      VpcCidr:
        default: Beacon Server VPC CIDR
      Subnet0Cidr:
        default: 1st Subnet Network
      Subnet1Cidr:
        default: 2nd Subnet Network
      AZ0:
        default: "1st Subnet AZ #"
      AZ1:
        default: "2nd Subnet AZ #"
      NodeSize:
        default: Node Requests/Min
      AutoScalingMinSize:
        default: Min Beacon Servers
      AutoScalingMaxSize:
        default: Max Beacon Servers

Mappings:
  Sizing:
    50K:
      InstanceType: t2.medium
      PacketsPerMinHigh: 200000
      PacketsPerMinLow: 80000
    100K:
      InstanceType: m5.large
      PacketsPerMinHigh: 500000
      PacketsPerMinLow: 200000
  SourceCode:
    General:
      S3Bucket: 'realtime-analytics-workshop'
      KeyPrefix: latest
  Solution:
    Data:
      ID: SO0038
      Version: 2.0.0
      SendAnonymousUsageData: 'False'
  DDB:
    Scaling:
      ReadTargetUtilization: 70
      ReadCapacityMin: 3
      ReadCapacityMax: 10
      WriteTargetUtilization: 50
      WriteCapacityMin: 30
      WriteCapacityMax: 1000

Conditions:
  CreateCWDashboard: !Equals [!Ref CWDashboard, Yes]

Resources:

# Beacon Server Networking
  VPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: !Ref VpcCidr
      EnableDnsSupport: true
      EnableDnsHostnames: true
      Tags:
        - Key: Name
          Value: !Sub '${AWS::StackName}-vpc'

  Subnet0:
    Type: AWS::EC2::Subnet
    Properties:
      MapPublicIpOnLaunch: true
      CidrBlock: !Ref Subnet0Cidr
      AvailabilityZone:
        !Select
          - !Ref AZ0
          - !GetAZs
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub '${AWS::StackName}-subnet-0'

  Subnet1:
    Type: AWS::EC2::Subnet
    Properties:
      MapPublicIpOnLaunch: true
      CidrBlock: !Ref Subnet1Cidr
      AvailabilityZone:
        !Select
          - !Ref AZ1
          - !GetAZs
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub '${AWS::StackName}-subnet-1'

  InternetGateway:
    Type: AWS::EC2::InternetGateway
    Properties:
      Tags:
        - Key: Name
          Value: !Sub '${AWS::StackName}-ig'

  GatewayAttachment:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      InternetGatewayId: !Ref InternetGateway
      VpcId: !Ref VPC

  RouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub '${AWS::StackName}-public-route-table'

  PublicRoute:
    Type: AWS::EC2::Route
    Properties:
      DestinationCidrBlock: "0.0.0.0/0"
      GatewayId: !Ref InternetGateway
      RouteTableId: !Ref RouteTable
    DependsOn: GatewayAttachment

  Subnet0RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref RouteTable
      SubnetId: !Ref Subnet0

  Subnet1RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref RouteTable
      SubnetId: !Ref Subnet1

# AutoScaling and LoadBalancing
  AutoScalingGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      VPCZoneIdentifier:
        - !Ref Subnet0
        - !Ref Subnet1
      LaunchConfigurationName: !Ref AutoScalingGroupLaunchConfig
      MinSize: !Ref AutoScalingMinSize
      MaxSize: !Ref AutoScalingMaxSize
      HealthCheckGracePeriod: 300
      HealthCheckType: ELB
      TargetGroupARNs:
        - !Ref TargetGroup
      Tags:
        - Key: "Name"
          Value: !Sub '${AWS::StackName}-beacon-web-server'
          PropagateAtLaunch: true
    CreationPolicy:
      ResourceSignal:
        Timeout: PT15M
    UpdatePolicy:
      AutoScalingRollingUpdate:
        MinInstancesInService: 1
        MaxBatchSize: 1
        PauseTime: PT15M
        SuspendProcesses:
          - HealthCheck
          - ReplaceUnhealthy
          - AZRebalance
          - AlarmNotification
          - ScheduledActions
        WaitOnResourceSignals: true

  PacketsAlarmHigh:
    Type: AWS::CloudWatch::Alarm
    Properties:
      EvaluationPeriods: 3
      Statistic: Average
      Threshold: !FindInMap [Sizing, !Ref NodeSize, PacketsPerMinHigh]
      AlarmDescription: Alarm if Network Packets In too high or metric disappears indicating instance is under high load
      Period: 60
      AlarmActions:
        - !Ref ScaleUpPolicy
      Namespace: AWS/EC2
      Dimensions:
        - Name: AutoScalingGroupName
          Value: !Ref AutoScalingGroup
      ComparisonOperator: GreaterThanThreshold
      MetricName: NetworkPacketsIn

  ScaleUpPolicy:
    Type: AWS::AutoScaling::ScalingPolicy
    Properties:
      AdjustmentType: ChangeInCapacity
      Cooldown: '300'
      AutoScalingGroupName: !Ref AutoScalingGroup
      ScalingAdjustment: 1

  PacketsAlarmLow:
    Type: AWS::CloudWatch::Alarm
    Properties:
      EvaluationPeriods: 3
      Statistic: Average
      Threshold: !FindInMap [Sizing, !Ref NodeSize, PacketsPerMinLow]
      AlarmDescription: Alarm if Network Packets In are too low and indicating that we can shed instances
      Period: 60
      AlarmActions:
        - !Ref ScaleDownPolicy
      Namespace: AWS/EC2
      Dimensions:
        - Name: AutoScalingGroupName
          Value: !Ref AutoScalingGroup
      ComparisonOperator: LessThanThreshold
      MetricName: NetworkPacketsIn

  ScaleDownPolicy:
    Type: AWS::AutoScaling::ScalingPolicy
    Properties:
      AdjustmentType: ChangeInCapacity
      Cooldown: '180'
      AutoScalingGroupName: !Ref AutoScalingGroup
      ScalingAdjustment: -1

  AutoScalingGroupLaunchConfig:
    Type: AWS::AutoScaling::LaunchConfiguration
    Metadata:
      AWS::CloudFormation::Init:
        config:
          packages:
            yum:
              httpd: []
              aws-kinesis-agent: []
          files:
            /etc/aws-kinesis/agent.json:
              content: !Sub |
                { "cloudwatch.emitMetrics": false,
                  "maxBufferAgeMillis":"1000",
                  "firehose.endpoint": "https://firehose.${AWS::Region}.amazonaws.com",
                  "flows": [
                    {
                     "filePattern": "/var/log/httpd/access_log*",
                      "deliveryStream": "${DeliveryStream}",
                      "partitionKeyOption": "RANDOM",
                      "dataProcessingOptions": [
                      {
                        "optionName": "LOGTOJSON",
                        "logFormat":"COMBINEDAPACHELOG",
                        "matchPattern": "^([\\d.]+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(.+?)\" (\\d{3}) ([0-9]+) \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\"$",
                        "customFieldNames": ["host", "ident", "authuser", "datetime", "request", "response", "bytes", "referrer", "agent", "event", "clientid", "page", "custom_metric_name", "custom_metric_int_value", "custom_metric_float_value", "custom_metric_string_value"]
                      }
                      ]
                    }
                  ]
                }
            /var/www/html/index.html:
              content: |-
                <HTML><HEAD></HEAD>
                <BODY>AWS Web Analytics Solution Accelerator</BODY>
                </HTML>
            /var/log/httpd/remove_old_log.sh:
              content: |-
                #!/bin/bash
                sleep 60
                echo "Rotating log on `date`" >> /var/log/httpd/rotate.log
                echo "New Log File: $1, Old Log File: $2" >> /var/log/httpd/rotate.log
                echo "rm $2" >> /var/log/httpd/rotate.log
                rm $2
              mode: "000775"
              owner: "root"
              group: "root"
          commands:
            aa-add-yum-update:
              command: yum update -y
            ab1-add-httpd:
              command: yum install httpd -y
            ab2-add-httpd:
              command: yum install aws-kinesis-agent -y
            ac1-add-service-httpd:
              command: chkconfig --add httpd
            ac2-make-readable-logs:
              command: chmod -R go+rX /var/log/httpd
            ad-add-service-aws-kinesis-agent:
              command: chkconfig --add aws-kinesis-agent
            ae-add-service-startup-aws-kinesis-agent:
              command: chkconfig aws-kinesis-agent on
            af-add-service-startup-httpd:
              command: chkconfig httpd on
            ba-return200s:
              command: echo "<Directory \"/\">" >> /etc/httpd/conf/httpd.conf
            bb-return200s:
              command: !Join ["", ["echo '   Header set Access-Control-Allow-Origin \"", Ref: "CorsOrig", "\"' >> /etc/httpd/conf/httpd.conf"]]
            bc-return200s:
              command: echo "   RewriteEngine on" >> /etc/httpd/conf/httpd.conf
            bd-return200s:
              command: echo "   RewriteRule '^.*' '/var/www/html/index.html'" >> /etc/httpd/conf/httpd.conf
            be-return200s:
              command: echo "</Directory>" >> /etc/httpd/conf/httpd.conf
            be-rotate-logs:
              command: sed -i 's/CustomLog logs\/access_log combined/CustomLog \"|\/usr\/sbin\/rotatelogs -p \/var\/log\/httpd\/remove_old_log.sh \/var\/log\/httpd\/access_log 1G\" combined/' /etc/httpd/conf/httpd.conf
            ca-add-data-header:
              command: sed -i 's/LogFormat "%h %l %u %t \\"%r\\" %>s %b \\"%{Referer}i\\"
                \\"%{User-Agent}i\\"" combined/LogFormat "%h %l %u %t \\"%r\\" %>s
                %b \\"%{Referer}i\\" \\"%{User-Agent}i\\" \\"%{event}i\\" \\"%{clientid}i\\"
                \\"%{page}i\\" \\"%{custom_metric_name}i\\" \\"%{custom_metric_int_value}i\\" \\"%{custom_metric_float_value}i\\" \\"%{custom_metric_string_value}i\\"" combined/' /etc/httpd/conf/httpd.conf
          services:
            sysvinit:
              httpd:
                enabled: 'true'
                ensureRunning: 'true'
                files:
                  - /etc/init.d/httpd
              aws-kinesis-agent:
                enabled: 'true'
                ensureRunning: 'true'
                files:
                  - /etc/init.d/aws-kinesis-agent
    Properties:
      ImageId: !Ref LatestAmiId
      InstanceType: !FindInMap [Sizing, !Ref NodeSize, InstanceType]
      IamInstanceProfile: !Ref WebServerInstanceProfile
      SecurityGroups:
        - !Ref WebServerNonSSLSecurityGroup
      UserData: !Base64
        Fn::Sub: |
          #!/bin/bash -xe
          /opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource AutoScalingGroupLaunchConfig --region ${AWS::Region}
          /opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource AutoScalingGroup
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            VolumeSize: 8

  PatchingAssociation:
    Type: AWS::SSM::Association
    Properties:
      AssociationName: WASAPatching
      Name: AWS-RunPatchBaseline
      Parameters:
        Operation:
          - Install
      ScheduleExpression: rate(1 day)
      Targets:
        - Key: tag:aws:cloudformation:stack-name
          Values:
            - !Ref AWS::StackName

  ApplicationLoadBalancer:
    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
    Properties:
      Scheme: internet-facing
      Subnets:
        - !Ref Subnet0
        - !Ref Subnet1
      SecurityGroups:
        - !GetAtt ALBSecurityGroup.GroupId

  ALBListener:
    Type: AWS::ElasticLoadBalancingV2::Listener
    Properties:
      DefaultActions:
        - Type: forward
          TargetGroupArn: !Ref TargetGroup
      LoadBalancerArn: !Ref ApplicationLoadBalancer
      Port: 80
      Protocol: HTTP

  TargetGroup:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      HealthCheckIntervalSeconds: 30
      HealthCheckPath: /
      HealthCheckTimeoutSeconds: 5
      Matcher:
        HttpCode: 200-499
      Port: 80
      Protocol: HTTP
      UnhealthyThresholdCount: 5
      TargetType: instance
      VpcId: !Ref VPC
      Tags:
        - Key: 'Name'
          Value: !Sub '${AWS::StackName}-target-group'

  ALBSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Web traffic for ALB
      VpcId: !Ref VPC
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 80
          ToPort: 80
          CidrIp: !Ref BeaconSourceCIDR

  WebServerNonSSLSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Web port only (80)
      VpcId: !Ref VPC
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 80
          ToPort: 80
          SourceSecurityGroupId: !Ref ALBSecurityGroup
      SecurityGroupEgress:
        - IpProtocol: '-1'
          CidrIp: 0.0.0.0/0


  WebServerInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /WebServerInstanceProfile/
      Roles:
        - !Ref WebServerKinesisRole

  WebServerKinesisRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: puttofirehose
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - firehose:PutRecord
                  - firehose:PutRecordBatch
                Resource:
                  - !GetAtt 'DeliveryStream.Arn'
        - PolicyName: ssmagent
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - ssm:DescribeAssociation
                  - ssm:GetDeployablePatchSnapshotForInstance
                  - ssm:GetDocument
                  - ssm:GetManifest
                  - ssm:GetParameters
                  - ssm:ListAssociations
                  - ssm:ListInstanceAssociations
                  - ssm:PutInventory
                  - ssm:PutComplianceItems
                  - ssm:PutConfigurePackageResult
                  - ssm:UpdateAssociationStatus
                  - ssm:UpdateInstanceAssociationStatus
                  - ssm:UpdateInstanceInformation
                  - ssmmessages:CreateControlChannel
                  - ssmmessages:CreateDataChannel
                  - ssmmessages:OpenControlChannel
                  - ssmmessages:OpenDataChannel
                Resource: '*'
              - Effect: Allow
                Action:
                  - s3:GetEncryptionConfiguration
                Resource: '*'
              - Effect: Allow
                Action:
                  - ec2messages:AcknowledgeMessage
                  - ec2messages:DeleteMessage
                  - ec2messages:FailMessage
                  - ec2messages:GetEndpoint
                  - ec2messages:GetMessages
                  - ec2messages:SendReply
                Resource: '*'
              - Effect: Allow
                Action:
                  - cloudwatch:PutMetricData
                Resource: '*'
              - Effect: Allow
                Action:
                  - ec2:DescribeInstanceStatus
                Resource: '*'
                Condition:
                  StringEquals:
                    ec2:ResourceTag/tag:aws:cloudformation:stack-name:
                      Ref: AWS::StackName
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:DescribeLogGroups
                  - logs:DescribeLogStreams
                  - logs:PutLogEvents
                Resource: '*'

# Kinesis Firehose
  DeliveryStreamRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - firehose.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: s3Access
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Sid: ''
                Effect: Allow
                Action:
                  - s3:AbortMultipartUpload
                  - s3:GetBucketLocation
                  - s3:GetObject
                  - s3:ListBucket
                  - s3:ListBucketMultipartUploads
                  - s3:PutObject
                Resource:
                  - !Sub '${AnalyticsBucket.Arn}'
                  - !Sub '${AnalyticsBucket.Arn}/*'
              - Sid: ''
                Effect: Allow
                Action:
                  - logs:PutLogEvents
                Resource:
                  - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/kinesisfirehose/*:log-stream:*'

  DeliveryStream:
    Type: AWS::KinesisFirehose::DeliveryStream
    Properties:
      DeliveryStreamType: DirectPut
      S3DestinationConfiguration:
        BucketARN: !Sub '${AnalyticsBucket.Arn}'
        BufferingHints:
          IntervalInSeconds: 60
          SizeInMBs: 1
        CompressionFormat: UNCOMPRESSED
        RoleARN: !GetAtt 'DeliveryStreamRole.Arn'

  AnalyticsBucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
  
# Kinesis Analytics Application
  KinesisAnalyticsRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - kinesisanalytics.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
      - PolicyName: kinesisaccess
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Sid: ReadInputStream
            Effect: Allow
            Action:
            - firehose:DescribeDeliveryStream
            - firehose:Get*
            Resource:
            - !GetAtt 'DeliveryStream.Arn'
          - Sid: LisInputStream
            Effect: Allow
            Action:
            - firehose:ListDeliveryStreams
            Resource: '*'
          - Sid: InvokeLambda
            Effect: Allow
            Action:
            - lambda:InvokeFunction
            - lambda:GetFunctionConfiguration
            Resource: !Sub
            - ${Param1}*
            - Param1: !GetAtt ProcessMetricsFunction.Arn

  KinesisAnalyticsApp:
    Type: AWS::KinesisAnalytics::Application
    Properties:
      ApplicationName: !Sub '${AWS::StackName}-WebMetricsApplication'
      ApplicationDescription: Kineis Analytics Web Metrics Solution Accelerator
      ApplicationCode: !Sub |
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



module-2/2-kinesis-analytics-module-update.yaml [1:690]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
AWSTemplateFormatVersion: '2010-09-09'
Description: "Module 2 - adding Kinesis Analytics, Processing Lambda, and DyanmoDB persistence"

Parameters:
  BeaconSourceCIDR:
    Type: String
    Description: "IP address CIDR block that send events to the Beacon Servers. This must allow web site user browsers to access the beacon servers."
    Default: "0.0.0.0/0"
    MinLength: 9
    MaxLength: 18
    AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
    ConstraintDescription: "must be a valid IP CIDR range of the form x.x.x.x/x."
  CorsOrig:
    Type: String
    Description: "This value will be returned by the API in the Access-Control-Allow-Origin header. A star (*) value will support any origin. We recommend specifying a specific origin (e.g. http://example.domain) to restrict cross-site access to your API."
    Default: "*"
  VpcCidr:
    Type: String
    Description: "CIDR block for new VPC for analytics beacon servers."
    Default: 10.0.0.0/23
  Subnet0Cidr:
    Type: String
    Description: "CIDR block for new VPC public subnet #1."
    Default: 10.0.0.0/24
  Subnet1Cidr:
    Type: String
    Description: "CIDR block for new VPC public subnet #2."
    Default: 10.0.1.0/24
  AZ0:
    Type: String
    Description: "Optional: Availability Zone number for public subnet #1."
    Default: 0
    AllowedValues:
      - 0
      - 1
      - 2
      - 3
      - 4
      - 5
  AZ1:
    Type: String
    Description: "Optional: Availability Zone number for public subnet #2."
    Default: 1
    AllowedValues:
      - 0
      - 1
      - 2
      - 3
      - 4
      - 5
  NodeSize:
    Type: String
    Description: "Number of requests per minute that each beacon node will support (50K = t2.medium, 100K = m5.large)."
    Default: 50K
    AllowedValues:
      - 50K
      - 100K
  AutoScalingMinSize:
    Type: Number
    Description: "Minimum number of beacon webservers (must be at least 2 for high availability)."
    Default: 2
    MinValue: 1
  AutoScalingMaxSize:
    Type: Number
    Description: "Maximum number of beacon webservers."
    Default: 6
    MinValue: 1
  CWDashboard:
    Type: String
    Description: "Create a CloudWatch dashboard for beacon server operational metrics?"
    Default: Yes
    AllowedValues:
      - Yes
      - No
  LatestAmiId:
    Type: 'AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>'
    Default: '/aws/service/ami-amazon-linux-latest/amzn-ami-hvm-x86_64-gp2'

Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: "Beacon Web Server Configuration"
        Parameters:
          - CWDashboard
          - BeaconSourceCIDR
          - CorsOrig
          - NodeSize
          - AutoScalingMinSize
          - AutoScalingMaxSize
      - Label:
          default: "Optional: VPC Configuration"
        Parameters:
          - VpcCidr
          - Subnet0Cidr
          - Subnet1Cidr
          - AZ0
          - AZ1
    ParameterLabels:
      CWDashboard:
        default: CloudWatch Dashboard
      BeaconSourceCIDR:
        default: Enable beacon events from
      CorsOrig:
        default: CORS Origin
      VpcCidr:
        default: Beacon Server VPC CIDR
      Subnet0Cidr:
        default: 1st Subnet Network
      Subnet1Cidr:
        default: 2nd Subnet Network
      AZ0:
        default: "1st Subnet AZ #"
      AZ1:
        default: "2nd Subnet AZ #"
      NodeSize:
        default: Node Requests/Min
      AutoScalingMinSize:
        default: Min Beacon Servers
      AutoScalingMaxSize:
        default: Max Beacon Servers

Mappings:
  Sizing:
    50K:
      InstanceType: t2.medium
      PacketsPerMinHigh: 200000
      PacketsPerMinLow: 80000
    100K:
      InstanceType: m5.large
      PacketsPerMinHigh: 500000
      PacketsPerMinLow: 200000
  SourceCode:
    General:
      S3Bucket: 'realtime-analytics-workshop'
      KeyPrefix: latest
  Solution:
    Data:
      ID: SO0038
      Version: 2.0.0
      SendAnonymousUsageData: 'False'
  DDB:
    Scaling:
      ReadTargetUtilization: 70
      ReadCapacityMin: 3
      ReadCapacityMax: 10
      WriteTargetUtilization: 50
      WriteCapacityMin: 30
      WriteCapacityMax: 1000

Conditions:
  CreateCWDashboard: !Equals [!Ref CWDashboard, Yes]

Resources:

# Beacon Server Networking
  VPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: !Ref VpcCidr
      EnableDnsSupport: true
      EnableDnsHostnames: true
      Tags:
        - Key: Name
          Value: !Sub '${AWS::StackName}-vpc'

  Subnet0:
    Type: AWS::EC2::Subnet
    Properties:
      MapPublicIpOnLaunch: true
      CidrBlock: !Ref Subnet0Cidr
      AvailabilityZone:
        !Select
          - !Ref AZ0
          - !GetAZs
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub '${AWS::StackName}-subnet-0'

  Subnet1:
    Type: AWS::EC2::Subnet
    Properties:
      MapPublicIpOnLaunch: true
      CidrBlock: !Ref Subnet1Cidr
      AvailabilityZone:
        !Select
          - !Ref AZ1
          - !GetAZs
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub '${AWS::StackName}-subnet-1'

  InternetGateway:
    Type: AWS::EC2::InternetGateway
    Properties:
      Tags:
        - Key: Name
          Value: !Sub '${AWS::StackName}-ig'

  GatewayAttachment:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      InternetGatewayId: !Ref InternetGateway
      VpcId: !Ref VPC

  RouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub '${AWS::StackName}-public-route-table'

  PublicRoute:
    Type: AWS::EC2::Route
    Properties:
      DestinationCidrBlock: "0.0.0.0/0"
      GatewayId: !Ref InternetGateway
      RouteTableId: !Ref RouteTable
    DependsOn: GatewayAttachment

  Subnet0RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref RouteTable
      SubnetId: !Ref Subnet0

  Subnet1RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref RouteTable
      SubnetId: !Ref Subnet1

# AutoScaling and LoadBalancing
  AutoScalingGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      VPCZoneIdentifier:
        - !Ref Subnet0
        - !Ref Subnet1
      LaunchConfigurationName: !Ref AutoScalingGroupLaunchConfig
      MinSize: !Ref AutoScalingMinSize
      MaxSize: !Ref AutoScalingMaxSize
      HealthCheckGracePeriod: 300
      HealthCheckType: ELB
      TargetGroupARNs:
        - !Ref TargetGroup
      Tags:
        - Key: "Name"
          Value: !Sub '${AWS::StackName}-beacon-web-server'
          PropagateAtLaunch: true
    CreationPolicy:
      ResourceSignal:
        Timeout: PT15M
    UpdatePolicy:
      AutoScalingRollingUpdate:
        MinInstancesInService: 1
        MaxBatchSize: 1
        PauseTime: PT15M
        SuspendProcesses:
          - HealthCheck
          - ReplaceUnhealthy
          - AZRebalance
          - AlarmNotification
          - ScheduledActions
        WaitOnResourceSignals: true

  PacketsAlarmHigh:
    Type: AWS::CloudWatch::Alarm
    Properties:
      EvaluationPeriods: 3
      Statistic: Average
      Threshold: !FindInMap [Sizing, !Ref NodeSize, PacketsPerMinHigh]
      AlarmDescription: Alarm if Network Packets In too high or metric disappears indicating instance is under high load
      Period: 60
      AlarmActions:
        - !Ref ScaleUpPolicy
      Namespace: AWS/EC2
      Dimensions:
        - Name: AutoScalingGroupName
          Value: !Ref AutoScalingGroup
      ComparisonOperator: GreaterThanThreshold
      MetricName: NetworkPacketsIn

  ScaleUpPolicy:
    Type: AWS::AutoScaling::ScalingPolicy
    Properties:
      AdjustmentType: ChangeInCapacity
      Cooldown: '300'
      AutoScalingGroupName: !Ref AutoScalingGroup
      ScalingAdjustment: 1

  PacketsAlarmLow:
    Type: AWS::CloudWatch::Alarm
    Properties:
      EvaluationPeriods: 3
      Statistic: Average
      Threshold: !FindInMap [Sizing, !Ref NodeSize, PacketsPerMinLow]
      AlarmDescription: Alarm if Network Packets In are too low and indicating that we can shed instances
      Period: 60
      AlarmActions:
        - !Ref ScaleDownPolicy
      Namespace: AWS/EC2
      Dimensions:
        - Name: AutoScalingGroupName
          Value: !Ref AutoScalingGroup
      ComparisonOperator: LessThanThreshold
      MetricName: NetworkPacketsIn

  ScaleDownPolicy:
    Type: AWS::AutoScaling::ScalingPolicy
    Properties:
      AdjustmentType: ChangeInCapacity
      Cooldown: '180'
      AutoScalingGroupName: !Ref AutoScalingGroup
      ScalingAdjustment: -1

  AutoScalingGroupLaunchConfig:
    Type: AWS::AutoScaling::LaunchConfiguration
    Metadata:
      AWS::CloudFormation::Init:
        config:
          packages:
            yum:
              httpd: []
              aws-kinesis-agent: []
          files:
            /etc/aws-kinesis/agent.json:
              content: !Sub |
                { "cloudwatch.emitMetrics": false,
                  "maxBufferAgeMillis":"1000",
                  "firehose.endpoint": "https://firehose.${AWS::Region}.amazonaws.com",
                  "flows": [
                    {
                     "filePattern": "/var/log/httpd/access_log*",
                      "deliveryStream": "${DeliveryStream}",
                      "partitionKeyOption": "RANDOM",
                      "dataProcessingOptions": [
                      {
                        "optionName": "LOGTOJSON",
                        "logFormat":"COMBINEDAPACHELOG",
                        "matchPattern": "^([\\d.]+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(.+?)\" (\\d{3}) ([0-9]+) \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\" \"(.+?)\"$",
                        "customFieldNames": ["host", "ident", "authuser", "datetime", "request", "response", "bytes", "referrer", "agent", "event", "clientid", "page", "custom_metric_name", "custom_metric_int_value", "custom_metric_float_value", "custom_metric_string_value"]
                      }
                      ]
                    }
                  ]
                }
            /var/www/html/index.html:
              content: |-
                <HTML><HEAD></HEAD>
                <BODY>AWS Web Analytics Solution Accelerator</BODY>
                </HTML>
            /var/log/httpd/remove_old_log.sh:
              content: |-
                #!/bin/bash
                sleep 60
                echo "Rotating log on `date`" >> /var/log/httpd/rotate.log
                echo "New Log File: $1, Old Log File: $2" >> /var/log/httpd/rotate.log
                echo "rm $2" >> /var/log/httpd/rotate.log
                rm $2
              mode: "000775"
              owner: "root"
              group: "root"
          commands:
            aa-add-yum-update:
              command: yum update -y
            ab1-add-httpd:
              command: yum install httpd -y
            ab2-add-httpd:
              command: yum install aws-kinesis-agent -y
            ac1-add-service-httpd:
              command: chkconfig --add httpd
            ac2-make-readable-logs:
              command: chmod -R go+rX /var/log/httpd
            ad-add-service-aws-kinesis-agent:
              command: chkconfig --add aws-kinesis-agent
            ae-add-service-startup-aws-kinesis-agent:
              command: chkconfig aws-kinesis-agent on
            af-add-service-startup-httpd:
              command: chkconfig httpd on
            ba-return200s:
              command: echo "<Directory \"/\">" >> /etc/httpd/conf/httpd.conf
            bb-return200s:
              command: !Join ["", ["echo '   Header set Access-Control-Allow-Origin \"", Ref: "CorsOrig", "\"' >> /etc/httpd/conf/httpd.conf"]]
            bc-return200s:
              command: echo "   RewriteEngine on" >> /etc/httpd/conf/httpd.conf
            bd-return200s:
              command: echo "   RewriteRule '^.*' '/var/www/html/index.html'" >> /etc/httpd/conf/httpd.conf
            be-return200s:
              command: echo "</Directory>" >> /etc/httpd/conf/httpd.conf
            be-rotate-logs:
              command: sed -i 's/CustomLog logs\/access_log combined/CustomLog \"|\/usr\/sbin\/rotatelogs -p \/var\/log\/httpd\/remove_old_log.sh \/var\/log\/httpd\/access_log 1G\" combined/' /etc/httpd/conf/httpd.conf
            ca-add-data-header:
              command: sed -i 's/LogFormat "%h %l %u %t \\"%r\\" %>s %b \\"%{Referer}i\\"
                \\"%{User-Agent}i\\"" combined/LogFormat "%h %l %u %t \\"%r\\" %>s
                %b \\"%{Referer}i\\" \\"%{User-Agent}i\\" \\"%{event}i\\" \\"%{clientid}i\\"
                \\"%{page}i\\" \\"%{custom_metric_name}i\\" \\"%{custom_metric_int_value}i\\" \\"%{custom_metric_float_value}i\\" \\"%{custom_metric_string_value}i\\"" combined/' /etc/httpd/conf/httpd.conf
          services:
            sysvinit:
              httpd:
                enabled: 'true'
                ensureRunning: 'true'
                files:
                  - /etc/init.d/httpd
              aws-kinesis-agent:
                enabled: 'true'
                ensureRunning: 'true'
                files:
                  - /etc/init.d/aws-kinesis-agent
    Properties:
      ImageId: !Ref LatestAmiId
      InstanceType: !FindInMap [Sizing, !Ref NodeSize, InstanceType]
      IamInstanceProfile: !Ref WebServerInstanceProfile
      SecurityGroups:
        - !Ref WebServerNonSSLSecurityGroup
      UserData: !Base64
        Fn::Sub: |
          #!/bin/bash -xe
          /opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource AutoScalingGroupLaunchConfig --region ${AWS::Region}
          /opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource AutoScalingGroup
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            VolumeSize: 8

  PatchingAssociation:
    Type: AWS::SSM::Association
    Properties:
      AssociationName: WASAPatching
      Name: AWS-RunPatchBaseline
      Parameters:
        Operation:
          - Install
      ScheduleExpression: rate(1 day)
      Targets:
        - Key: tag:aws:cloudformation:stack-name
          Values:
            - !Ref AWS::StackName

  ApplicationLoadBalancer:
    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
    Properties:
      Scheme: internet-facing
      Subnets:
        - !Ref Subnet0
        - !Ref Subnet1
      SecurityGroups:
        - !GetAtt ALBSecurityGroup.GroupId

  ALBListener:
    Type: AWS::ElasticLoadBalancingV2::Listener
    Properties:
      DefaultActions:
        - Type: forward
          TargetGroupArn: !Ref TargetGroup
      LoadBalancerArn: !Ref ApplicationLoadBalancer
      Port: 80
      Protocol: HTTP

  TargetGroup:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      HealthCheckIntervalSeconds: 30
      HealthCheckPath: /
      HealthCheckTimeoutSeconds: 5
      Matcher:
        HttpCode: 200-499
      Port: 80
      Protocol: HTTP
      UnhealthyThresholdCount: 5
      TargetType: instance
      VpcId: !Ref VPC
      Tags:
        - Key: 'Name'
          Value: !Sub '${AWS::StackName}-target-group'

  ALBSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Web traffic for ALB
      VpcId: !Ref VPC
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 80
          ToPort: 80
          CidrIp: !Ref BeaconSourceCIDR

  WebServerNonSSLSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Web port only (80)
      VpcId: !Ref VPC
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 80
          ToPort: 80
          SourceSecurityGroupId: !Ref ALBSecurityGroup
      SecurityGroupEgress:
        - IpProtocol: '-1'
          CidrIp: 0.0.0.0/0

  WebServerInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /WebServerInstanceProfile/
      Roles:
        - !Ref WebServerKinesisRole

  WebServerKinesisRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: puttofirehose
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - firehose:PutRecord
                  - firehose:PutRecordBatch
                Resource:
                  - !GetAtt 'DeliveryStream.Arn'
        - PolicyName: ssmagent
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - ssm:DescribeAssociation
                  - ssm:GetDeployablePatchSnapshotForInstance
                  - ssm:GetDocument
                  - ssm:GetManifest
                  - ssm:GetParameters
                  - ssm:ListAssociations
                  - ssm:ListInstanceAssociations
                  - ssm:PutInventory
                  - ssm:PutComplianceItems
                  - ssm:PutConfigurePackageResult
                  - ssm:UpdateAssociationStatus
                  - ssm:UpdateInstanceAssociationStatus
                  - ssm:UpdateInstanceInformation
                  - ssmmessages:CreateControlChannel
                  - ssmmessages:CreateDataChannel
                  - ssmmessages:OpenControlChannel
                  - ssmmessages:OpenDataChannel
                Resource: '*'
              - Effect: Allow
                Action:
                  - s3:GetEncryptionConfiguration
                Resource: '*'
              - Effect: Allow
                Action:
                  - ec2messages:AcknowledgeMessage
                  - ec2messages:DeleteMessage
                  - ec2messages:FailMessage
                  - ec2messages:GetEndpoint
                  - ec2messages:GetMessages
                  - ec2messages:SendReply
                Resource: '*'
              - Effect: Allow
                Action:
                  - cloudwatch:PutMetricData
                Resource: '*'
              - Effect: Allow
                Action:
                  - ec2:DescribeInstanceStatus
                Resource: '*'
                Condition:
                  StringEquals:
                    ec2:ResourceTag/tag:aws:cloudformation:stack-name:
                      Ref: AWS::StackName
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:DescribeLogGroups
                  - logs:DescribeLogStreams
                  - logs:PutLogEvents
                Resource: '*'

# Kinesis Firehose
  DeliveryStreamRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - firehose.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: s3Access
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Sid: ''
                Effect: Allow
                Action:
                  - s3:AbortMultipartUpload
                  - s3:GetBucketLocation
                  - s3:GetObject
                  - s3:ListBucket
                  - s3:ListBucketMultipartUploads
                  - s3:PutObject
                Resource:
                  - !Sub '${AnalyticsBucket.Arn}'
                  - !Sub '${AnalyticsBucket.Arn}/*'
              - Sid: ''
                Effect: Allow
                Action:
                  - logs:PutLogEvents
                Resource:
                  - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/kinesisfirehose/*:log-stream:*'

  DeliveryStream:
    Type: AWS::KinesisFirehose::DeliveryStream
    Properties:
      DeliveryStreamType: DirectPut
      S3DestinationConfiguration:
        BucketARN: !Sub '${AnalyticsBucket.Arn}'
        BufferingHints:
          IntervalInSeconds: 60
          SizeInMBs: 1
        CompressionFormat: UNCOMPRESSED
        RoleARN: !GetAtt 'DeliveryStreamRole.Arn'

  AnalyticsBucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
  
# Kinesis Analytics Application
  KinesisAnalyticsRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - kinesisanalytics.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
      - PolicyName: kinesisaccess
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Sid: ReadInputStream
            Effect: Allow
            Action:
            - firehose:DescribeDeliveryStream
            - firehose:Get*
            Resource:
            - !GetAtt 'DeliveryStream.Arn'
          - Sid: LisInputStream
            Effect: Allow
            Action:
            - firehose:ListDeliveryStreams
            Resource: '*'
          - Sid: InvokeLambda
            Effect: Allow
            Action:
            - lambda:InvokeFunction
            - lambda:GetFunctionConfiguration
            Resource: !Sub
            - ${Param1}*
            - Param1: !GetAtt ProcessMetricsFunction.Arn

  KinesisAnalyticsApp:
    Type: AWS::KinesisAnalytics::Application
    Properties:
      ApplicationName: !Sub '${AWS::StackName}-WebMetricsApplication'
      ApplicationDescription: Kineis Analytics Web Metrics Solution Accelerator
      ApplicationCode: !Sub |
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



