cvat-aws-all.yaml [791:1156]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            - /mnt/
          MountPoints:
            - ContainerPath: /mnt
              SourceVolume: cvat-share
          Image: amazon/aws-cli
          Essential: False
          Name: dataset-s3-container2
        - LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Ref LogGroup
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: ecs
          EntryPoint:
            - bash
            - "-c"
          Command: 
            - >
              tar --owner=1000 --group=1000 --numeric-owner -zxvf /mnt/handball.tar.gz -C /mnt/ &&
              rm /mnt/handball.tar.gz
          DependsOn:
            - Condition: SUCCESS
              ContainerName: dataset-s3-container1
            - Condition: SUCCESS
              ContainerName: dataset-s3-container2
          MountPoints:
            - ContainerPath: /mnt
              SourceVolume: cvat-share
          Image: ubuntu:20.04
          Name: dataset-tar-container
      Memory: '2048'
      TaskRoleArn: !GetAtt EcsTaskRole.Arn
      Family: dataset-cmd-task
      RequiresCompatibilities:
        - FARGATE
      NetworkMode: awsvpc
      Cpu: '1024'
      Volumes:
        - EFSVolumeConfiguration:
            FilesystemId: !Ref CVATEfs
            AuthorizationConfig:
              AccessPointId: !Ref ShareAccessPoint
            TransitEncryption: ENABLED
            RootDirectory: "/"
          Name: cvat-share

  LambdaExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      Path: /
      Policies:
        - PolicyName: root
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:PutLogEvents'
                Resource: '*'
              - Effect: Allow
                Action:
                  - ecs:RunTask
                  - iam:PassRole
                Resource: '*'

  RunCmdFunction:
    Type: AWS::Lambda::Function
    Properties:
      Code:
        ZipFile: |
          import boto3
          import urllib3
          import json
          SUCCESS = "SUCCESS"
          FAILED = "FAILED"
          http = urllib3.PoolManager()
          def lambda_handler(event, context):
              print(event)
              if (event['RequestType'] == 'Create'):
                  try:
                      client = boto3.client('ecs')
                      print("Running task.")
                      response = client.run_task(
                          cluster=event['ResourceProperties']['Cluster'], 
                          launchType='FARGATE',
                          taskDefinition=event['ResourceProperties']['TaskDefinition'],  # <-- notice no revision number
                          count=1,
                          platformVersion='LATEST',
                          networkConfiguration={
                              'awsvpcConfiguration': {
                                  'subnets': event['ResourceProperties']['Subnets'],
                              'securityGroups': [event['ResourceProperties']['securityGroups']],
                              'assignPublicIp': 'DISABLED'
                              }
                          })
                      print("Finished invoking task.")
                  except Exception as e:
                      send(event, context, FAILED, {'exception': str(e)})
              send(event, context, SUCCESS, dict())


          def send(event, context, responseStatus, responseData, physicalResourceId=None, noEcho=False, reason=None):
              responseUrl = event['ResponseURL']

              print(responseUrl)

              responseBody = {
                  'Status' : responseStatus,
                  'Reason' : reason or "See the details in CloudWatch Log Stream: {}".format(context.log_stream_name),
                  'PhysicalResourceId' : physicalResourceId or context.log_stream_name,
                  'StackId' : event['StackId'],
                  'RequestId' : event['RequestId'],
                  'LogicalResourceId' : event['LogicalResourceId'],
                  'NoEcho' : noEcho,
                  'Data' : responseData
              }

              json_responseBody = json.dumps(responseBody)

              print("Response body:")
              print(json_responseBody)

              headers = {
                  'content-type' : '',
                  'content-length' : str(len(json_responseBody))
              }

              try:
                  response = http.request('PUT', responseUrl, headers=headers, body=json_responseBody)
                  print("Status code:", response.status)


              except Exception as e:
                  print("send failed executing:", e)

      Handler: index.lambda_handler
      Role: !GetAtt LambdaExecutionRole.Arn
      Runtime: python3.6
      Timeout: '300'

  CvatCmdCustom:
    Type: Custom::CvatCmd
    DependsOn: CmdTaskDefinition
    Properties:
      ServiceToken: !GetAtt RunCmdFunction.Arn
      Region: !Ref AWS::Region
      Cluster: !Ref ECSCluster
      TaskDefinition: cvat-cmd-task
      Subnets:
        - !Ref PrivateSubnet1
        - !Ref PrivateSubnet2
        - !Ref PrivateSubnet3
      securityGroups: !Ref AppServerSecurityGroup

  DatasetCmdCustom:
    Type: Custom::DatasetCmd
    DependsOn: DatasetTaskDefinition
    Properties:
      ServiceToken: !GetAtt RunCmdFunction.Arn
      Region: !Ref AWS::Region
      Cluster: !Ref ECSCluster
      TaskDefinition: dataset-cmd-task
      Subnets:
        - !Ref PrivateSubnet1
        - !Ref PrivateSubnet2
        - !Ref PrivateSubnet3
      securityGroups: !Ref AppServerSecurityGroup


  CvatApplicationLoadBalancer:
    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
    Properties:
      Name: cvat-alb
      Scheme: internet-facing
      SecurityGroups:
        - !Ref ALBServerSecurityGroup
      Subnets:
        - !Ref PublicSubnet1
        - !Ref PublicSubnet2
        - !Ref PublicSubnet3
      Type: application

  CvatUITargetGroup:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      Name: cvat-ui-target
      Protocol: HTTP
      TargetType: ip
      VpcId: !Ref VPC
      Port: 80
      TargetGroupAttributes:
        - Key: deregistration_delay.timeout_seconds
          Value: 60

  CvatServerTargetGroup:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      Name: cvat-server-target
      Protocol: HTTP
      Matcher:
        HttpCode: "300-399"
      HealthCheckPath: "/api/swagger"
      TargetType: ip
      VpcId: !Ref VPC
      Port: 80
      TargetGroupAttributes:
        - Key: deregistration_delay.timeout_seconds
          Value: 60


  CvatLoadBalancerListener:
    Type: AWS::ElasticLoadBalancingV2::Listener
    Properties:
      LoadBalancerArn: !Ref CvatApplicationLoadBalancer
      Port: 80
      Protocol: HTTP
      DefaultActions:
        - Type: forward
          TargetGroupArn: !Ref CvatUITargetGroup

  CvatServerListenerRule1:
    Type: AWS::ElasticLoadBalancingV2::ListenerRule
    Properties:
      Priority: 100
      ListenerArn: !Ref CvatLoadBalancerListener
      Actions:
        - Type: forward
          TargetGroupArn: !Ref CvatServerTargetGroup
      Conditions:
        - Field: path-pattern
          Values:
            - "/api/*"
            - "/git/*"
            - "/opencv/*"
            - "/analytics/*"
            - "/static/*"

  CvatServerListenerRule2:
    Type: AWS::ElasticLoadBalancingV2::ListenerRule
    Properties:
      Priority: 200
      ListenerArn: !Ref CvatLoadBalancerListener
      Actions:
        - Type: forward
          TargetGroupArn: !Ref CvatServerTargetGroup
      Conditions:
        - Field: path-pattern
          Values:
            - "/admin*"
            - "/documentation/*"
            - "/django-rq*"

  ECSCluster:
    Type: AWS::ECS::Cluster
    Properties:
      ClusterName: CvatFargateCluster
      CapacityProviders:
        - FARGATE
        - FARGATE_SPOT
      DefaultCapacityProviderStrategy:
        - CapacityProvider: FARGATE
          Weight: 1
        - CapacityProvider: FARGATE_SPOT
          Weight: 1

  ServerECSService:
    Type: AWS::ECS::Service
    DependsOn: 
      - CvatLoadBalancerListener
      - CvatCmdCustom
    Properties:
      ServiceName: cvat-server-service
      HealthCheckGracePeriodSeconds: 300
      LaunchType: FARGATE
      LoadBalancers:
        - ContainerName: cvat-server-container
          ContainerPort: 8080
          TargetGroupArn: !Ref CvatServerTargetGroup
      Cluster: !Ref ECSCluster
      DesiredCount: 2
      TaskDefinition: !Ref ServerTaskDefinition
      NetworkConfiguration:
        AwsvpcConfiguration:
          SecurityGroups:
            - !Ref AppServerSecurityGroup
          Subnets:
            - !Ref PrivateSubnet1
            - !Ref PrivateSubnet2
            - !Ref PrivateSubnet3

  UIECSService:
    Type: AWS::ECS::Service
    DependsOn: CvatLoadBalancerListener
    Properties:
      ServiceName: cvat-ui-service
      LaunchType: FARGATE
      LoadBalancers:
        - ContainerName: cvat-ui-container
          ContainerPort: 80
          TargetGroupArn: !Ref CvatUITargetGroup
      Cluster: !Ref ECSCluster
      DesiredCount: 2
      TaskDefinition: !Ref UITaskDefinition
      NetworkConfiguration:
        AwsvpcConfiguration:
          SecurityGroups:
            - !Ref UIServerSecurityGroup
          Subnets:
            - !Ref PrivateSubnet1
            - !Ref PrivateSubnet2
            - !Ref PrivateSubnet3

  ServerlessECSService:
    Type: AWS::ECS::Service
    Properties:
      ServiceName: cvat-serverless-service
      LaunchType: FARGATE
      Cluster: !Ref ECSCluster
      DesiredCount: 2
      TaskDefinition: !Ref ServerlessTaskDefinition
      ServiceRegistries:
        - RegistryArn: !GetAtt CavtServerlessServiceDiscoveryEntry.Arn
      NetworkConfiguration:
        AwsvpcConfiguration:
          SecurityGroups:
            - !Ref ServerlessSecurityGroup
          Subnets:
            - !Ref PrivateSubnet1
            - !Ref PrivateSubnet2
            - !Ref PrivateSubnet3

  CvatCloudmapServiceDiscovery:
    Type: AWS::ServiceDiscovery::PrivateDnsNamespace
    Properties:
      Name: cvat.local
      Description: CVAT Service Discovery
      Vpc: !Ref VPC

  CavtServerlessServiceDiscoveryEntry:
        Type: AWS::ServiceDiscovery::Service
        Properties:
            Name: serverless
            DnsConfig:
                DnsRecords: 
                    - Type: A
                      TTL: "10"
                NamespaceId: !Ref CvatCloudmapServiceDiscovery
            HealthCheckCustomConfig:
                FailureThreshold: '1'

  CvatSagemakerRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: CvatSagemakerExecRole
      ManagedPolicyArns: 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



cvat-aws-ap-northeast-1-all.yaml [791:1156]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            - /mnt/
          MountPoints:
            - ContainerPath: /mnt
              SourceVolume: cvat-share
          Image: amazon/aws-cli
          Essential: False
          Name: dataset-s3-container2
        - LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Ref LogGroup
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: ecs
          EntryPoint:
            - bash
            - "-c"
          Command: 
            - >
              tar --owner=1000 --group=1000 --numeric-owner -zxvf /mnt/handball.tar.gz -C /mnt/ &&
              rm /mnt/handball.tar.gz
          DependsOn:
            - Condition: SUCCESS
              ContainerName: dataset-s3-container1
            - Condition: SUCCESS
              ContainerName: dataset-s3-container2
          MountPoints:
            - ContainerPath: /mnt
              SourceVolume: cvat-share
          Image: ubuntu:20.04
          Name: dataset-tar-container
      Memory: '2048'
      TaskRoleArn: !GetAtt EcsTaskRole.Arn
      Family: dataset-cmd-task
      RequiresCompatibilities:
        - FARGATE
      NetworkMode: awsvpc
      Cpu: '1024'
      Volumes:
        - EFSVolumeConfiguration:
            FilesystemId: !Ref CVATEfs
            AuthorizationConfig:
              AccessPointId: !Ref ShareAccessPoint
            TransitEncryption: ENABLED
            RootDirectory: "/"
          Name: cvat-share

  LambdaExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      Path: /
      Policies:
        - PolicyName: root
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:PutLogEvents'
                Resource: '*'
              - Effect: Allow
                Action:
                  - ecs:RunTask
                  - iam:PassRole
                Resource: '*'

  RunCmdFunction:
    Type: AWS::Lambda::Function
    Properties:
      Code:
        ZipFile: |
          import boto3
          import urllib3
          import json
          SUCCESS = "SUCCESS"
          FAILED = "FAILED"
          http = urllib3.PoolManager()
          def lambda_handler(event, context):
              print(event)
              if (event['RequestType'] == 'Create'):
                  try:
                      client = boto3.client('ecs')
                      print("Running task.")
                      response = client.run_task(
                          cluster=event['ResourceProperties']['Cluster'], 
                          launchType='FARGATE',
                          taskDefinition=event['ResourceProperties']['TaskDefinition'],  # <-- notice no revision number
                          count=1,
                          platformVersion='LATEST',
                          networkConfiguration={
                              'awsvpcConfiguration': {
                                  'subnets': event['ResourceProperties']['Subnets'],
                              'securityGroups': [event['ResourceProperties']['securityGroups']],
                              'assignPublicIp': 'DISABLED'
                              }
                          })
                      print("Finished invoking task.")
                  except Exception as e:
                      send(event, context, FAILED, {'exception': str(e)})
              send(event, context, SUCCESS, dict())


          def send(event, context, responseStatus, responseData, physicalResourceId=None, noEcho=False, reason=None):
              responseUrl = event['ResponseURL']

              print(responseUrl)

              responseBody = {
                  'Status' : responseStatus,
                  'Reason' : reason or "See the details in CloudWatch Log Stream: {}".format(context.log_stream_name),
                  'PhysicalResourceId' : physicalResourceId or context.log_stream_name,
                  'StackId' : event['StackId'],
                  'RequestId' : event['RequestId'],
                  'LogicalResourceId' : event['LogicalResourceId'],
                  'NoEcho' : noEcho,
                  'Data' : responseData
              }

              json_responseBody = json.dumps(responseBody)

              print("Response body:")
              print(json_responseBody)

              headers = {
                  'content-type' : '',
                  'content-length' : str(len(json_responseBody))
              }

              try:
                  response = http.request('PUT', responseUrl, headers=headers, body=json_responseBody)
                  print("Status code:", response.status)


              except Exception as e:
                  print("send failed executing:", e)

      Handler: index.lambda_handler
      Role: !GetAtt LambdaExecutionRole.Arn
      Runtime: python3.6
      Timeout: '300'

  CvatCmdCustom:
    Type: Custom::CvatCmd
    DependsOn: CmdTaskDefinition
    Properties:
      ServiceToken: !GetAtt RunCmdFunction.Arn
      Region: !Ref AWS::Region
      Cluster: !Ref ECSCluster
      TaskDefinition: cvat-cmd-task
      Subnets:
        - !Ref PrivateSubnet1
        - !Ref PrivateSubnet2
        - !Ref PrivateSubnet3
      securityGroups: !Ref AppServerSecurityGroup

  DatasetCmdCustom:
    Type: Custom::DatasetCmd
    DependsOn: DatasetTaskDefinition
    Properties:
      ServiceToken: !GetAtt RunCmdFunction.Arn
      Region: !Ref AWS::Region
      Cluster: !Ref ECSCluster
      TaskDefinition: dataset-cmd-task
      Subnets:
        - !Ref PrivateSubnet1
        - !Ref PrivateSubnet2
        - !Ref PrivateSubnet3
      securityGroups: !Ref AppServerSecurityGroup


  CvatApplicationLoadBalancer:
    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
    Properties:
      Name: cvat-alb
      Scheme: internet-facing
      SecurityGroups:
        - !Ref ALBServerSecurityGroup
      Subnets:
        - !Ref PublicSubnet1
        - !Ref PublicSubnet2
        - !Ref PublicSubnet3
      Type: application

  CvatUITargetGroup:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      Name: cvat-ui-target
      Protocol: HTTP
      TargetType: ip
      VpcId: !Ref VPC
      Port: 80
      TargetGroupAttributes:
        - Key: deregistration_delay.timeout_seconds
          Value: 60

  CvatServerTargetGroup:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      Name: cvat-server-target
      Protocol: HTTP
      Matcher:
        HttpCode: "300-399"
      HealthCheckPath: "/api/swagger"
      TargetType: ip
      VpcId: !Ref VPC
      Port: 80
      TargetGroupAttributes:
        - Key: deregistration_delay.timeout_seconds
          Value: 60


  CvatLoadBalancerListener:
    Type: AWS::ElasticLoadBalancingV2::Listener
    Properties:
      LoadBalancerArn: !Ref CvatApplicationLoadBalancer
      Port: 80
      Protocol: HTTP
      DefaultActions:
        - Type: forward
          TargetGroupArn: !Ref CvatUITargetGroup

  CvatServerListenerRule1:
    Type: AWS::ElasticLoadBalancingV2::ListenerRule
    Properties:
      Priority: 100
      ListenerArn: !Ref CvatLoadBalancerListener
      Actions:
        - Type: forward
          TargetGroupArn: !Ref CvatServerTargetGroup
      Conditions:
        - Field: path-pattern
          Values:
            - "/api/*"
            - "/git/*"
            - "/opencv/*"
            - "/analytics/*"
            - "/static/*"

  CvatServerListenerRule2:
    Type: AWS::ElasticLoadBalancingV2::ListenerRule
    Properties:
      Priority: 200
      ListenerArn: !Ref CvatLoadBalancerListener
      Actions:
        - Type: forward
          TargetGroupArn: !Ref CvatServerTargetGroup
      Conditions:
        - Field: path-pattern
          Values:
            - "/admin*"
            - "/documentation/*"
            - "/django-rq*"

  ECSCluster:
    Type: AWS::ECS::Cluster
    Properties:
      ClusterName: CvatFargateCluster
      CapacityProviders:
        - FARGATE
        - FARGATE_SPOT
      DefaultCapacityProviderStrategy:
        - CapacityProvider: FARGATE
          Weight: 1
        - CapacityProvider: FARGATE_SPOT
          Weight: 1

  ServerECSService:
    Type: AWS::ECS::Service
    DependsOn: 
      - CvatLoadBalancerListener
      - CvatCmdCustom
    Properties:
      ServiceName: cvat-server-service
      HealthCheckGracePeriodSeconds: 300
      LaunchType: FARGATE
      LoadBalancers:
        - ContainerName: cvat-server-container
          ContainerPort: 8080
          TargetGroupArn: !Ref CvatServerTargetGroup
      Cluster: !Ref ECSCluster
      DesiredCount: 2
      TaskDefinition: !Ref ServerTaskDefinition
      NetworkConfiguration:
        AwsvpcConfiguration:
          SecurityGroups:
            - !Ref AppServerSecurityGroup
          Subnets:
            - !Ref PrivateSubnet1
            - !Ref PrivateSubnet2
            - !Ref PrivateSubnet3

  UIECSService:
    Type: AWS::ECS::Service
    DependsOn: CvatLoadBalancerListener
    Properties:
      ServiceName: cvat-ui-service
      LaunchType: FARGATE
      LoadBalancers:
        - ContainerName: cvat-ui-container
          ContainerPort: 80
          TargetGroupArn: !Ref CvatUITargetGroup
      Cluster: !Ref ECSCluster
      DesiredCount: 2
      TaskDefinition: !Ref UITaskDefinition
      NetworkConfiguration:
        AwsvpcConfiguration:
          SecurityGroups:
            - !Ref UIServerSecurityGroup
          Subnets:
            - !Ref PrivateSubnet1
            - !Ref PrivateSubnet2
            - !Ref PrivateSubnet3

  ServerlessECSService:
    Type: AWS::ECS::Service
    Properties:
      ServiceName: cvat-serverless-service
      LaunchType: FARGATE
      Cluster: !Ref ECSCluster
      DesiredCount: 2
      TaskDefinition: !Ref ServerlessTaskDefinition
      ServiceRegistries:
        - RegistryArn: !GetAtt CavtServerlessServiceDiscoveryEntry.Arn
      NetworkConfiguration:
        AwsvpcConfiguration:
          SecurityGroups:
            - !Ref ServerlessSecurityGroup
          Subnets:
            - !Ref PrivateSubnet1
            - !Ref PrivateSubnet2
            - !Ref PrivateSubnet3

  CvatCloudmapServiceDiscovery:
    Type: AWS::ServiceDiscovery::PrivateDnsNamespace
    Properties:
      Name: cvat.local
      Description: CVAT Service Discovery
      Vpc: !Ref VPC

  CavtServerlessServiceDiscoveryEntry:
        Type: AWS::ServiceDiscovery::Service
        Properties:
            Name: serverless
            DnsConfig:
                DnsRecords: 
                    - Type: A
                      TTL: "10"
                NamespaceId: !Ref CvatCloudmapServiceDiscovery
            HealthCheckCustomConfig:
                FailureThreshold: '1'

  CvatSagemakerRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: CvatSagemakerExecRole
      ManagedPolicyArns: 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



