handlers/sf-datalake-export/cfn.yaml

AWSTemplateFormatVersion: "2010-09-09"
Description: export salesforce data to the data lake

Parameters:
  Stage:
    Description: Stage name
    Type: String
    AllowedValues:
    - CODE
    - PROD
    Default: CODE
Conditions:
  IsProd: !Equals [!Ref "Stage", "PROD"]
Mappings:
  StageMap:
    CODE:
      destBuckets: "arn:aws:s3:::gu-salesforce-export-code*"
      localbucketName : "gu-salesforce-export-code"
    PROD:
      destBuckets: ["arn:aws:s3:::gu-salesforce-export-prod*", "arn:aws:s3:::ophan-raw-salesforce-*"]
      localbucketName: "gu-salesforce-export-prod"
Resources:
  localBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: !FindInMap [StageMap, !Ref Stage, localbucketName]
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      LifecycleConfiguration:
        Rules:
          - TagFilters:
              - Key: housekeeping
                Value: delete
            ExpirationInDays: 2
            Status: Enabled
  StartJobRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: /
      Policies:
      - PolicyName: LambdaPolicy
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - logs:CreateLogGroup
            - logs:CreateLogStream
            - logs:PutLogEvents
            - lambda:InvokeFunction
            Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/sf-start-export-job-${Stage}:log-stream:*"
      - PolicyName: ReadPrivateCredentials
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action: s3:GetObject
            Resource: !Sub "arn:aws:s3:::gu-reader-revenue-private/membership/support-service-lambdas/${Stage}/sfExportAuth-${Stage}*.json"
  StartJob:
    Type: AWS::Lambda::Function
    Properties:
      Description: start a sf export job
      FunctionName:
        !Sub sf-start-export-job-${Stage}
      Code:
        S3Bucket: support-service-lambdas-dist
        S3Key: !Sub membership/${Stage}/sf-datalake-export/sf-datalake-export.jar
      Handler: com.gu.sf_datalake_export.handlers.StartJobHandler::apply
      Environment:
        Variables:
          Stage: !Ref Stage
      Role:
        !GetAtt StartJobRole.Arn
      MemorySize: 1536
      Runtime: java21
      Timeout: 300
      Architectures:
        - arm64
    DependsOn:
    - StartJobRole
  BatchStateRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: /
      Policies:
      - PolicyName: LambdaPolicy
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - logs:CreateLogGroup
            - logs:CreateLogStream
            - logs:PutLogEvents
            - lambda:InvokeFunction
            Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/sf-get-batch-state-${Stage}:log-stream:*"
      - PolicyName: ReadPrivateCredentials
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action: s3:GetObject
            Resource: !Sub "arn:aws:s3:::gu-reader-revenue-private/membership/support-service-lambdas/${Stage}/sfExportAuth-${Stage}*.json"
  GetBatchesLambda:
    Type: AWS::Lambda::Function
    Properties:
      Description: get state of batches related to an salesforce bulk api query
      FunctionName:
        !Sub sf-get-batch-state-${Stage}
      Code:
        S3Bucket: support-service-lambdas-dist
        S3Key: !Sub membership/${Stage}/sf-datalake-export/sf-datalake-export.jar
      Handler: com.gu.sf_datalake_export.handlers.GetBatchesHandler::apply
      Environment:
        Variables:
          Stage: !Ref Stage
      Role:
        !GetAtt BatchStateRole.Arn
      MemorySize: 1536
      Runtime: java21
      Timeout: 300
      Architectures:
        - arm64
    DependsOn:
    - BatchStateRole
  DownloadBatchRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: /
      Policies:
      - PolicyName: LambdaPolicy
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - logs:CreateLogGroup
            - logs:CreateLogStream
            - logs:PutLogEvents
            - lambda:InvokeFunction
            Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/sf-download-batch-${Stage}:log-stream:*"
      - PolicyName: ReadPrivateCredentials
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action: s3:GetObject
            Resource: !Sub "arn:aws:s3:::gu-reader-revenue-private/membership/support-service-lambdas/${Stage}/sfExportAuth-${Stage}*.json"
      - PolicyName: DestBuckets
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - s3:PutObject
            - s3:PutObjectAcl
            Resource: !FindInMap [StageMap, !Ref Stage, destBuckets]
  CleanBucketRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      Path: /
      Policies:
        - PolicyName: LambdaPolicy
          PolicyDocument:
            Statement:
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                  - lambda:InvokeFunction
                Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/sf-export-clean-bucket-${Stage}:log-stream:*"
        - PolicyName: DestBucket
          PolicyDocument:
            Statement:
              - Effect: Allow
                Action:
                  - s3:DeleteObject
                  - s3:GetObject
                  - s3:GetObjectAcl
                  - s3:GetBucketAcl
                  - s3:ListBucket
                  - s3:GetObjectVersion
                  - s3:DeleteObjectVersion
                Resource: !FindInMap [StageMap, !Ref Stage, destBuckets]

  DownloadBatch:
    Type: AWS::Lambda::Function
    Properties:
      Description: download result of completed bulk api batches into s3
      FunctionName:
        !Sub sf-download-batch-${Stage}
      Code:
        S3Bucket: support-service-lambdas-dist
        S3Key: !Sub membership/${Stage}/sf-datalake-export/sf-datalake-export.jar
      Handler: com.gu.sf_datalake_export.handlers.DownloadBatchHandler::apply
      Environment:
        Variables:
          Stage: !Ref Stage
      Role:
        !GetAtt DownloadBatchRole.Arn
      MemorySize: 1792
      Runtime: java21
      Timeout: 900
      Architectures:
        - arm64
    DependsOn:
    - DownloadBatchRole
  EndJobRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: /
      Policies:
      - PolicyName: LambdaPolicy
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - logs:CreateLogGroup
            - logs:CreateLogStream
            - logs:PutLogEvents
            - lambda:InvokeFunction
            Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/sf-end-export-job-${Stage}:log-stream:*"
      - PolicyName: ReadPrivateCredentials
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action: s3:GetObject
            Resource: !Sub "arn:aws:s3:::gu-reader-revenue-private/membership/support-service-lambdas/${Stage}/sfExportAuth-${Stage}*.json"
      - PolicyName: DestBuckets
        PolicyDocument:
          Statement:
            - Effect: Allow
              Action:
                - s3:PutObject
                - s3:PutObjectAcl
              Resource: !FindInMap [StageMap, !Ref Stage, destBuckets]
  CleanBucket:
    Type: AWS::Lambda::Function
    Properties:
      Description: clean old results from bucket before uploading new ones
      FunctionName:
        !Sub sf-export-clean-bucket-${Stage}
      Code:
        S3Bucket: support-service-lambdas-dist
        S3Key: !Sub membership/${Stage}/sf-datalake-export/sf-datalake-export.jar
      Handler: com.gu.sf_datalake_export.handlers.CleanBucketHandler::apply
      Environment:
        Variables:
          Stage: !Ref Stage
      Role:
        !GetAtt CleanBucketRole.Arn
      MemorySize: 1536
      Runtime: java21
      Timeout: 900
      Architectures:
        - arm64
    DependsOn:
      - CleanBucketRole
  EndJob:
    Type: AWS::Lambda::Function
    Properties:
      Description: close a sf export job
      FunctionName:
        !Sub sf-end-export-job-${Stage}
      Code:
        S3Bucket: support-service-lambdas-dist
        S3Key: !Sub membership/${Stage}/sf-datalake-export/sf-datalake-export.jar
      Handler: com.gu.sf_datalake_export.handlers.EndJobHandler::apply
      Environment:
        Variables:
          Stage: !Ref Stage
      Role:
        !GetAtt EndJobRole.Arn
      MemorySize: 1536
      Runtime: java21
      Timeout: 300
      Architectures:
        - arm64
    DependsOn:
    - EndJobRole
  StatesExecutionRole:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          Effect: Allow
          Principal:
            Service: !Sub 'states.${AWS::Region}.amazonaws.com'
          Action: 'sts:AssumeRole'
      Path: /
      Policies:
      - PolicyName: StatesExecutionPolicy
        PolicyDocument:
          Version: 2012-10-17
          Statement:
          - Effect: Allow
            Action:
            - 'lambda:InvokeFunction'
            Resource: '*'
  StateMachine:
    Type: 'AWS::StepFunctions::StateMachine'
    Properties:
      StateMachineName: !Sub 'salesforce-export-${Stage}'
      RoleArn: !GetAtt
      - StatesExecutionRole
      - Arn
      DefinitionString: !Sub
      - |-
            {
              "StartAt": "StartJob",
              "States": {
                "StartJob": {
                  "Type": "Task",
                  "Resource": "${startJobArn}",
                  "Next": "WaitSomeTime",
                  "Retry": [{
                              "ErrorEquals": ["States.ALL"],
                              "IntervalSeconds": 30,
                              "MaxAttempts": 3
                            }]
                },
                "WaitSomeTime": {
                  "Type": "Wait",
                  "Seconds": 5,
                  "Next": "getBatches"
                },
                "getBatches": {
                  "Type": "Task",
                  "Resource": "${getBatchesArn}",
                  "Next": "checkPendingBatches",
                  "Retry": [{
                              "ErrorEquals": ["States.ALL"],
                              "IntervalSeconds": 30,
                              "MaxAttempts": 3
                            }]
                },
                "checkPendingBatches": {
                  "Type": "Choice",
                  "Choices": [{
                                "Variable": "$.jobStatus",
                                "StringEquals": "Completed",
                                "Next": "cleanBucket"
                              }],
                  "Default": "WaitSomeTime"
                },
                "cleanBucket": {
                "Type": "Task",
                "Resource": "${cleanBucketArn}",
                "Next": "downloadBatch",
                "Retry": [{
                      "ErrorEquals": ["States.ALL"],
                      "IntervalSeconds": 30,
                      "MaxAttempts": 3
                    }]
                },
                "downloadBatch": {
                  "Type": "Task",
                  "Resource": "${downloadBatchArn}",
                  "Next": "CheckPendingDownloads",
                  "Retry": [{
                              "ErrorEquals": ["States.ALL"],
                              "IntervalSeconds": 300,
                              "MaxAttempts": 3
                            }]
                },
                "CheckPendingDownloads": {
                  "Type": "Choice",
                  "Choices": [{
                                "Variable": "$.done",
                                "BooleanEquals": true,
                                "Next": "endJob"
                              }],
                  "Default": "downloadBatch"
                },
                "endJob": {
                  "Type": "Task",
                  "Resource": "${endJobArn}",
                  "Next": "done",
                  "Retry": [{
                              "ErrorEquals": ["States.ALL"],
                              "IntervalSeconds": 30,
                              "MaxAttempts": 3
                            }]
                },
                "done": {
                  "Type": "Pass",
                  "End": true
                }
              }
            }
      - {
        startJobArn: !GetAtt [ StartJob, Arn ],
        getBatchesArn: !GetAtt [ GetBatchesLambda, Arn ],
        downloadBatchArn: !GetAtt [ DownloadBatch, Arn ],
        endJobArn: !GetAtt [ EndJob, Arn ],
        cleanBucketArn: !GetAtt [ CleanBucket, Arn ]

      }

  TriggerRole:
    Type: AWS::IAM::Role
    Condition: IsProd
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - events.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: TriggerStateMchine
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - states:StartExecution
                Resource: !Ref StateMachine
  ScheduleRule1:
    Type: "AWS::Events::Rule"
    Condition: IsProd
    Properties:
      Description: "trigger salesforce export every day at 01:00 GMT"
      ScheduleExpression: "cron(0 1 ? * * *)"
      State: "ENABLED"
      Targets:
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-Contact-${Stage}
          Input: |
            {
             "objectName": "Contact"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-Subscription-${Stage}
          Input: |
            {
             "objectName": "Subscription"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-Case-${Stage}
          Input: |
            {
             "objectName": "Case"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-Discount-${Stage}
          Input: |
            {
             "objectName": "Discount"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-CardExpiry-${Stage}
          Input: |
            {
             "objectName": "CardExpiry"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
  ScheduleRule2:
    Type: "AWS::Events::Rule"
    Condition: IsProd
    Properties:
      Description: "trigger salesforce export every day at 01:00 GMT"
      ScheduleExpression: "cron(0 1 ? * * *)"
      State: "ENABLED"
      Targets:
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-Account-${Stage}
          Input: |
            {
             "objectName": "Account"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-DirectDebitMandateFailure-${Stage}
          Input: |
            {
             "objectName": "DirectDebitMandateFailure"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-DirectDebitMandate-${Stage}
          Input: |
            {
             "objectName": "DirectDebitMandate"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-DirectDebitMandateEvent-${Stage}
          Input: |
            {
             "objectName": "DirectDebitMandateEvent"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-CancellationSurvey-${Stage}
          Input: |
            {
             "objectName": "CancellationSurvey"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
  ScheduleRule3:
    Type: "AWS::Events::Rule"
    Condition: IsProd
    Properties:
      Description: "trigger salesforce export every day at 01:00 GMT"
      ScheduleExpression: "cron(0 1 ? * * *)"
      State: "ENABLED"
      Targets:
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-ImovoContract-${Stage}
          Input: |
            {
             "objectName": "ImovoContract"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-DigitalVoucher-${Stage}
          Input: |
            {
             "objectName": "DigitalVoucher"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]
        -
          Arn: !Ref StateMachine
          Id: !Sub trigger_sf_export-ZuoraSubscriptionProductFeature-${Stage}
          Input: |
            {
             "objectName": "ZuoraSubscriptionProductFeature"
            }
          RoleArn: !GetAtt [ TriggerRole, Arn ]