formstack-baton-requests/cloud-formation.yaml (694 lines of code) (raw):

AWSTemplateFormatVersion: "2010-09-09" Parameters: Stage: Description: Stage name Type: String AllowedValues: - PROD - CODE Default: CODE ResultsBucket: Description: Bucket where results and status updates are uploaded to Type: String Default: gu-baton-results VpcId: Description: Vpc where the lambda is being created Type: String VpcSubnets: Description: Subnets to use in VPC Type: CommaDelimitedList EncryptionPasswordPath: Type: String FormstackAccountOneTokenPath: Type: String FormstackAccountTwoTokenPath: Type: String BcryptSaltPath: Type: String LastUpdatedTableName: Type: String SubmissionsTableName: Type: String BatonAccountId: Type: String AlarmEmailAddress: Description: Contact email for alarms Type: String Conditions: IsProd: !Equals [ !Ref Stage, PROD ] Mappings: StageVariables: PROD: SubmissionsTableWriteCapacityUnits: 20 SubmissionsTableReadCapacityUnits: 5 CODE: SubmissionsTableWriteCapacityUnits: 1 SubmissionsTableReadCapacityUnits: 1 Resources: TopicSendEmail: Type: AWS::SNS::Topic Properties: DisplayName: SendEmailTopic Subscription: - Endpoint: !Ref 'AlarmEmailAddress' Protocol: email BatonAccountInvokeRole: Type: AWS::IAM::Role Properties: RoleName: !Sub "baton-formstack-lambda-role-${Stage}" AssumeRolePolicyDocument: Statement: - Effect: Allow Principal: AWS: !Sub "arn:aws:iam::${BatonAccountId}:root" Action: - sts:AssumeRole Path: / Policies: - PolicyName: LambdaPolicy PolicyDocument: Statement: - Effect: Allow Action: - lambda:InvokeFunction Resource: - !GetAtt FormstackBatonSarLambda.Arn - !GetAtt FormstackBatonRerLambda.Arn FormstackBatonSarLambdaRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: - sts:AssumeRole Path: / Policies: - PolicyName: EC2Policy PolicyDocument: Statement: - Effect: Allow Action: - ec2:CreateNetworkInterface - ec2:DescribeNetworkInterfaces - ec2:DeleteNetworkInterface Resource: "*" - PolicyName: LogPolicy PolicyDocument: Statement: - Effect: Allow Action: - logs:CreateLogGroup - logs:CreateLogStream - logs:PutLogEvents Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/formstack-baton-sar-lambda-${Stage}:log-stream:*" - PolicyName: S3ListPolicy PolicyDocument: Statement: - Effect: Allow Action: - s3:ListBucket Resource: !Sub arn:aws:s3:::${ResultsBucket} - PolicyName: InvokeStepFunctionPolicy PolicyDocument: Statement: - Effect: Allow Action: - states:StartExecution Resource: !Ref FormstackSar LambdaSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: !Sub Security group for the formstack-baton-requests-${Stage} lambdas VpcId: !Ref VpcId FormstackBatonSarLambda: Type: AWS::Lambda::Function Properties: Description: Initiates the PerformFormstackSarLambda and checks the status of Formstack SARs via S3 FunctionName: !Sub formstack-baton-sar-lambda-${Stage} Code: S3Bucket: identity-lambda S3Key: !Sub identity/${Stage}/formstack-baton-requests/main.jar Handler: com.gu.identity.formstackbatonrequests.Handler::handleSar Environment: Variables: RESULTS_BUCKET: !Ref ResultsBucket RESULTS_PATH: !Sub formstack-results/${Stage} STATE_MACHINE_ARN: !Ref FormstackSar MemorySize: 1024 Runtime: java11 Timeout: 120 VpcConfig: SecurityGroupIds: - !Ref LambdaSecurityGroup SubnetIds: !Ref VpcSubnets Role: !GetAtt FormstackBatonSarLambdaRole.Arn FormstackSubmissionIds: Type: AWS::DynamoDB::Table Properties: TableName: !Ref SubmissionsTableName DeletionProtectionEnabled: true AttributeDefinitions: - AttributeName: email AttributeType: S - AttributeName: submissionId AttributeType: S KeySchema: - AttributeName: email KeyType: HASH - AttributeName: submissionId KeyType: RANGE ProvisionedThroughput: ReadCapacityUnits: !FindInMap [StageVariables, !Ref 'Stage', SubmissionsTableReadCapacityUnits] WriteCapacityUnits: !FindInMap [StageVariables, !Ref 'Stage', SubmissionsTableWriteCapacityUnits] Tags: - Key: devx-backup-enabled Value: true FormstackSubmissionsLastUpdated: Type: AWS::DynamoDB::Table Properties: TableName: !Ref LastUpdatedTableName DeletionProtectionEnabled: true AttributeDefinitions: - AttributeName: formstackSubmissionTableMetadata AttributeType: S KeySchema: - AttributeName: formstackSubmissionTableMetadata KeyType: HASH ProvisionedThroughput: ReadCapacityUnits: 1 WriteCapacityUnits: 1 Tags: - Key: devx-backup-enabled Value: true PerformFormstackSarLambdaRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: - sts:AssumeRole Path: / Policies: - PolicyName: EC2Policy PolicyDocument: Statement: - Effect: Allow Action: - ec2:CreateNetworkInterface - ec2:DescribeNetworkInterfaces - ec2:DeleteNetworkInterface Resource: "*" - PolicyName: LogPolicy PolicyDocument: Statement: - Effect: Allow Action: - logs:CreateLogGroup - logs:CreateLogStream - logs:PutLogEvents Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/formstack-baton-perform-sar-lambda-${Stage}:log-stream:*" - PolicyName: S3ListPolicy PolicyDocument: Statement: - Effect: Allow Action: - s3:ListBucket Resource: !Sub arn:aws:s3:::${ResultsBucket} - PolicyName: S3WritePolicy PolicyDocument: Statement: - Effect: Allow Action: - s3:PutObject - s3:GetObject Resource: !Sub arn:aws:s3:::${ResultsBucket}/formstack-results/${Stage}/* - PolicyName: GetParamsPolicy PolicyDocument: Statement: - Effect: Allow Action: ssm:GetParameter Resource: - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${FormstackAccountOneTokenPath} - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${FormstackAccountTwoTokenPath} - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${EncryptionPasswordPath} - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${BcryptSaltPath} - PolicyName: SubmissionIdsTablePolicy PolicyDocument: Statement: - Effect: Allow Action: - dynamodb:Query Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${SubmissionsTableName} PerformFormstackSarLambda: Type: AWS::Lambda::Function Properties: Description: Performs SAR to Formstack, writing results to S3 FunctionName: !Sub formstack-baton-perform-sar-lambda-${Stage} Code: S3Bucket: identity-lambda S3Key: !Sub identity/${Stage}/formstack-baton-requests/main.jar Handler: com.gu.identity.formstackbatonrequests.Handler::handlePerformSar Environment: Variables: STAGE: !Ref Stage RESULTS_BUCKET: !Ref ResultsBucket RESULTS_PATH: !Sub formstack-results/${Stage} ENCRYPTION_PASSWORD_PATH: !Ref EncryptionPasswordPath FORMSTACK_ACCOUNT_ONE_TOKEN_PATH: !Ref FormstackAccountOneTokenPath FORMSTACK_ACCOUNT_TWO_TOKEN_PATH: !Ref FormstackAccountTwoTokenPath BCRYPT_SALT_PATH: !Ref BcryptSaltPath SUBMISSION_TABLE_NAME: !Ref SubmissionsTableName LAST_UPDATED_TABLE_NAME: !Ref LastUpdatedTableName MemorySize: 1024 Runtime: java11 Timeout: 900 VpcConfig: SecurityGroupIds: - !Ref LambdaSecurityGroup SubnetIds: !Ref VpcSubnets Role: !GetAtt PerformFormstackSarLambdaRole.Arn DependsOn: - PerformFormstackSarLambdaRole FormstackBatonRerLambdaRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: - sts:AssumeRole Path: / Policies: - PolicyName: EC2Policy PolicyDocument: Statement: - Effect: Allow Action: - ec2:CreateNetworkInterface - ec2:DescribeNetworkInterfaces - ec2:DeleteNetworkInterface Resource: "*" - PolicyName: LogPolicy PolicyDocument: Statement: - Effect: Allow Action: - logs:CreateLogGroup - logs:CreateLogStream - logs:PutLogEvents Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/formstack-baton-rer-lambda-${Stage}:log-stream:*" - PolicyName: S3ListPolicy PolicyDocument: Statement: - Effect: Allow Action: - s3:ListBucket Resource: !Sub arn:aws:s3:::${ResultsBucket} - PolicyName: InvokeStepFunctionPolicy PolicyDocument: Statement: - Effect: Allow Action: - states:StartExecution Resource: !Ref FormstackRer FormstackBatonRerLambda: Type: AWS::Lambda::Function Properties: Description: Initiates the PerformFormstackRerLambda and checks the status of Formstack RERs via S3 FunctionName: !Sub formstack-baton-rer-lambda-${Stage} Code: S3Bucket: identity-lambda S3Key: !Sub identity/${Stage}/formstack-baton-requests/main.jar Handler: com.gu.identity.formstackbatonrequests.Handler::handleRer Environment: Variables: RESULTS_BUCKET: !Ref ResultsBucket RESULTS_PATH: !Sub formstack-results/${Stage} STATE_MACHINE_ARN: !Ref FormstackRer MemorySize: 1024 Runtime: java11 Timeout: 120 VpcConfig: SecurityGroupIds: - !Ref LambdaSecurityGroup SubnetIds: !Ref VpcSubnets Role: !GetAtt FormstackBatonRerLambdaRole.Arn PerformFormstackRerLambdaRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: - sts:AssumeRole Path: / Policies: - PolicyName: EC2Policy PolicyDocument: Statement: - Effect: Allow Action: - ec2:CreateNetworkInterface - ec2:DescribeNetworkInterfaces - ec2:DeleteNetworkInterface Resource: "*" - PolicyName: LogPolicy PolicyDocument: Statement: - Effect: Allow Action: - logs:CreateLogGroup - logs:CreateLogStream - logs:PutLogEvents Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/formstack-baton-perform-rer-lambda-${Stage}:log-stream:*" - PolicyName: S3ListPolicy PolicyDocument: Statement: - Effect: Allow Action: - s3:ListBucket Resource: !Sub arn:aws:s3:::${ResultsBucket} - PolicyName: S3WritePolicy PolicyDocument: Statement: - Effect: Allow Action: - s3:PutObject - s3:GetObject Resource: !Sub arn:aws:s3:::${ResultsBucket}/formstack-results/${Stage}/* - PolicyName: GetParamsPolicy PolicyDocument: Statement: - Effect: Allow Action: ssm:GetParameter Resource: - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${FormstackAccountOneTokenPath} - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${FormstackAccountTwoTokenPath} - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${EncryptionPasswordPath} - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${BcryptSaltPath} - PolicyName: SubmissionIdsTablePolicy PolicyDocument: Statement: - Effect: Allow Action: - dynamodb:Query - dynamodb:DeleteItem Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${SubmissionsTableName} PerformFormstackRerLambda: Type: AWS::Lambda::Function Properties: Description: Performs RER to Formstack, writing status updates to S3 FunctionName: !Sub formstack-baton-perform-rer-lambda-${Stage} Code: S3Bucket: identity-lambda S3Key: !Sub identity/${Stage}/formstack-baton-requests/main.jar Handler: com.gu.identity.formstackbatonrequests.Handler::handlePerformRer Environment: Variables: STAGE: !Ref Stage RESULTS_BUCKET: !Ref ResultsBucket RESULTS_PATH: !Sub formstack-results/${Stage} ENCRYPTION_PASSWORD_PATH: !Ref EncryptionPasswordPath FORMSTACK_ACCOUNT_ONE_TOKEN_PATH: !Ref FormstackAccountOneTokenPath FORMSTACK_ACCOUNT_TWO_TOKEN_PATH: !Ref FormstackAccountTwoTokenPath BCRYPT_SALT_PATH: !Ref BcryptSaltPath SUBMISSION_TABLE_NAME: !Ref SubmissionsTableName LAST_UPDATED_TABLE_NAME: !Ref LastUpdatedTableName MemorySize: 1024 Runtime: java11 Timeout: 900 VpcConfig: SecurityGroupIds: - !Ref LambdaSecurityGroup SubnetIds: !Ref VpcSubnets Role: !GetAtt PerformFormstackRerLambdaRole.Arn DependsOn: - PerformFormstackRerLambdaRole UpdateDynamoLambdaRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: - sts:AssumeRole Path: / Policies: - PolicyName: EC2Policy PolicyDocument: Statement: - Effect: Allow Action: - ec2:CreateNetworkInterface - ec2:DescribeNetworkInterfaces - ec2:DeleteNetworkInterface Resource: "*" - PolicyName: LogPolicy PolicyDocument: Statement: - Effect: Allow Action: - logs:CreateLogGroup - logs:CreateLogStream - logs:PutLogEvents Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/formstack-baton-update-dynamo-lambda-${Stage}:log-stream:*" - PolicyName: S3WritePolicy PolicyDocument: Statement: - Effect: Allow Action: - s3:PutObject Resource: !Sub arn:aws:s3:::${ResultsBucket}/formstack-results/${Stage}/* - PolicyName: GetParamsPolicy PolicyDocument: Statement: - Effect: Allow Action: ssm:GetParameter Resource: - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${FormstackAccountOneTokenPath} - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${FormstackAccountTwoTokenPath} - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${EncryptionPasswordPath} - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${BcryptSaltPath} - PolicyName: SubmissionIdsTablePolicy PolicyDocument: Statement: - Effect: Allow Action: - dynamodb:BatchWriteItem Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${SubmissionsTableName} - PolicyName: LastUpdatedTablePolicy PolicyDocument: Statement: - Effect: Allow Action: - dynamodb:GetItem - dynamodb:PutItem Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${LastUpdatedTableName} UpdateDynamoLambda: Type: AWS::Lambda::Function Properties: Description: Updates Dynamo with recent submissions in Formstack FunctionName: !Sub formstack-baton-update-dynamo-lambda-${Stage} Code: S3Bucket: identity-lambda S3Key: !Sub identity/${Stage}/formstack-baton-requests/main.jar Handler: com.gu.identity.formstackbatonrequests.Handler::handleUpdateDynamo Environment: Variables: STAGE: !Ref Stage RESULTS_BUCKET: !Ref ResultsBucket RESULTS_PATH: !Sub formstack-results/${Stage} ENCRYPTION_PASSWORD_PATH: !Ref EncryptionPasswordPath FORMSTACK_ACCOUNT_ONE_TOKEN_PATH: !Ref FormstackAccountOneTokenPath FORMSTACK_ACCOUNT_TWO_TOKEN_PATH: !Ref FormstackAccountTwoTokenPath BCRYPT_SALT_PATH: !Ref BcryptSaltPath SUBMISSION_TABLE_NAME: !Ref SubmissionsTableName LAST_UPDATED_TABLE_NAME: !Ref LastUpdatedTableName MemorySize: 1024 Runtime: java11 Timeout: 900 VpcConfig: SecurityGroupIds: - !Ref LambdaSecurityGroup SubnetIds: !Ref VpcSubnets Role: !GetAtt UpdateDynamoLambdaRole.Arn DependsOn: - UpdateDynamoLambdaRole FormstackBatonRequestsLambdaErrorAlarm: Type: AWS::CloudWatch::Alarm Properties: AlarmName: !Sub 'formstack-baton-requests-lambda-${Stage} failed execution' AlarmDescription: Alert when formstack baton request lambdas error Namespace: AWS/Lambda Dimensions: - Name: BatonSarLambdaName Value: !Ref FormstackBatonSarLambda - Name: PerformSarLambdaName Value: !Ref PerformFormstackSarLambda MetricName: Errors Statistic: Sum ComparisonOperator: GreaterThanOrEqualToThreshold Threshold: '1' Period: '60' EvaluationPeriods: '1' AlarmActions: - !If [IsProd, !Ref 'TopicSendEmail', !Ref 'AWS::NoValue'] # **************************************************************************** # Step Function and Role # **************************************************************************** StatesExecutionRole: Type: AWS::IAM::Role Properties: RoleName: !Sub "formstack-baton-step-function-role-${Stage}" AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Sid: '' Effect: Allow Principal: Service: !Sub 'states.${AWS::Region}.amazonaws.com' Action: 'sts:AssumeRole' Policies: - PolicyName: "run-lambda" PolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Action: "lambda:InvokeFunction" Resource: - !GetAtt PerformFormstackSarLambda.Arn - !GetAtt UpdateDynamoLambda.Arn - !GetAtt PerformFormstackRerLambda.Arn DynamoUpdateStatesExecutionRole: Type: AWS::IAM::Role Properties: RoleName: !Sub "formstack-dynamo-update-step-function-role-${Stage}" AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Sid: '' Effect: Allow Principal: Service: !Sub 'states.${AWS::Region}.amazonaws.com' Action: 'sts:AssumeRole' Policies: - PolicyName: "run-lambda" PolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Action: "lambda:InvokeFunction" Resource: - !GetAtt UpdateDynamoLambda.Arn # commented out because a bug needs to be fixed before this is can be used and it's risky to use as it is (can lead to # submissions being permanently missed) # if the whole execution cannot finish in a single lambda run the maxUpdateSeconds parameter is lost in the next # lambda invocation so the "last updated date" in dynamo is brought up to the current timestamp even though submissions # in the forms from the first lambda execution were only fetched respecting the maxUpdateSeconds param # # # FormstackUpdateDynamo: # Type: "AWS::StepFunctions::StateMachine" # Properties: # StateMachineName: !Join ['-', ['FormstackUpdateDynamo', !Ref Stage]] # DefinitionString: # !Sub # - |- # { # "Comment": "Triggered manually if we need to update the Formstack dynamodb table in smaller increments. Used if the full update performed by the other state machines is so big that it causes errors.", # "StartAt": "AddParams", # "States": { # "AddParams": { # "Type": "Pass", # "Parameters": { # "timeOfStart.$": "States.ArrayGetItem(States.StringSplit($$.Execution.StartTime, 'Z'),0)", # "requestType": "SAR", # "initiationReference": "updateDynamo", # "subjectEmail": "unused_email@guardian.co.uk", # "dataProvider": "formstack", # "accountNumber": 1, # "formPage": 1, # "count": 25, # "maxUpdateSeconds.$": "$.maxUpdateSeconds" # }, # "Next": "UpdateDynamoWithAccountOne" # }, # "UpdateDynamoWithAccountOne": { # "Type": "Task", # "Resource": "${updateLambdaArn}", # "Next": "CheckAccountOneUpdateCompletion" # }, # "CheckAccountOneUpdateCompletion": { # "Type": "Choice", # "Choices": [ # { # "Variable": "$.status", # "StringEquals": "pending", # "Next": "UpdateDynamoWithAccountOne" # } # ], # "Default": "Done" # }, # # "Done": { # "Type": "Pass", # "End": true # } # } # } # - # updateLambdaArn: !GetAtt UpdateDynamoLambda.Arn # RoleArn: !GetAtt DynamoUpdateStatesExecutionRole.Arn FormstackSar: Type: "AWS::StepFunctions::StateMachine" Properties: StateMachineName: !Join ['-', ['FormstackSar', !Ref Stage]] DefinitionString: !Sub - |- { "Comment": "A step function for performing Formstack SARs", "StartAt": "AddAccountOneParam", "States": { "AddAccountOneParam": { "Type": "Pass", "Next": "UpdateDynamoWithAccountOne", "Result": "1", "ResultPath": "$.accountNumber" }, "UpdateDynamoWithAccountOne": { "Type": "Task", "Resource": "${updateLambdaArn}", "Next": "CheckAccountOneUpdateCompletion" }, "CheckAccountOneUpdateCompletion": { "Type": "Choice", "Choices": [ { "Variable": "$.status", "StringEquals": "pending", "Next": "UpdateDynamoWithAccountOne" } ], "Default": "PerformSar" }, "PerformSar": { "Type": "Task", "Resource": "${sarLambdaArn}", "End": true } } } - sarLambdaArn: !GetAtt PerformFormstackSarLambda.Arn updateLambdaArn: !GetAtt UpdateDynamoLambda.Arn RoleArn: !GetAtt StatesExecutionRole.Arn FormstackRer: Type: "AWS::StepFunctions::StateMachine" Properties: StateMachineName: !Join ['-', ['FormstackRer', !Ref Stage]] DefinitionString: !Sub - |- { "Comment": "A step function for performing Formstack RERs", "StartAt": "AddAccountOneParam", "States": { "AddAccountOneParam": { "Type": "Pass", "Next": "UpdateDynamoWithAccountOne", "Result": "1", "ResultPath": "$.accountNumber" }, "UpdateDynamoWithAccountOne": { "Type": "Task", "Resource": "${updateLambdaArn}", "Next": "CheckAccountOneUpdateCompletion" }, "CheckAccountOneUpdateCompletion": { "Type": "Choice", "Choices": [ { "Variable": "$.status", "StringEquals": "pending", "Next": "UpdateDynamoWithAccountOne" } ], "Default": "PerformRer" }, "PerformRer": { "Type": "Task", "Resource": "${rerLambdaArn}", "End": true } } } - rerLambdaArn: !GetAtt PerformFormstackRerLambda.Arn updateLambdaArn: !GetAtt UpdateDynamoLambda.Arn RoleArn: !GetAtt StatesExecutionRole.Arn