compute-nest-best-practice/opensource/kubernetes/template.yaml

ROSTemplateFormatVersion: "2015-09-01"
Description:
  zh-cn: 使用ROS模板部署Ack-Distro驱动的K8S多节点集群，含可选公网访问与自定义资源配置。
  en: Use ROS templates to deploy Ack-Distro-driven K8S multi-node clusters, including optional public network access and custom resource configuration.
Parameters:
  PayType:
    Type: String
    Label:
      en: ECS Instance Charge Type
      zh-cn: 付费类型
    Default: PostPaid
    AllowedValues:
      - PostPaid
      - PrePaid
    AssociationProperty: ChargeType
    AssociationPropertyMetadata:
      LocaleKey: InstanceChargeType
  PayPeriodUnit:
    Type: String
    Label:
      en: Pay Period Unit
      zh-cn: 购买资源时长周期
    Default: Month
    AllowedValues:
      - Month
      - Year
    AssociationProperty: PayPeriodUnit
    AssociationPropertyMetadata:
      Visible:
        Condition:
          Fn::Not:
            Fn::Equals:
              - ${PayType}
              - PostPaid
  PayPeriod:
    Type: Number
    Label:
      en: Period
      zh-cn: 购买资源时长
    Default: 1
    AllowedValues:
      - 1
      - 2
      - 3
      - 4
      - 5
      - 6
      - 7
      - 8
      - 9
    AssociationProperty: PayPeriod
    AssociationPropertyMetadata:
      Visible:
        Condition:
          Fn::Not:
            Fn::Equals:
              - ${PayType}
              - PostPaid
  ZoneId:
    Type: String
    AssociationProperty: ALIYUN::ECS::Instance::ZoneId
    Label:
      en: VSwitch Availability Zone
      zh-cn: 交换机可用区
  VpcId:
    AssociationProperty: ALIYUN::ECS::VPC::VPCId
    Type: String
    Description:
      en: Please search the ID starting with (vpc-xxx) from console-Virtual Private Cloud
      zh-cn: 现有虚拟专有网络的实例ID。
    Label:
      en: VPC ID
      zh-cn: 专有网络VPC实例ID
    AssociationPropertyMetadata:
      Visible:
        Condition:
          Fn::Equals:
            - ${AutoCreateVpcAndVsw}
            - false
  VSwitchId:
    AssociationProperty: ALIYUN::ECS::VSwitch::VSwitchId
    AssociationPropertyMetadata:
      VpcId: ${VpcId}
      ZoneId: ${ZoneId}
      Visible:
        Condition:
          Fn::Equals:
            - ${AutoCreateVpcAndVsw}
            - false
    Type: String
    Description:
      en: Instance ID of existing business network switches, console-Virtual Private Cloud-VSwitches under query
      zh-cn: 现有业务网络交换机的实例ID。
    Label:
      en: VSwitch ID
      zh-cn: 交换机实例ID
  LoadBalancerSpec:
    Type: String
    AssociationProperty: ALIYUN::SLB::Instance::InstanceType
    Label:
      en: LoadBalancer Specifications
      zh-cn: 负载均衡实例规格
    Description:
      en: By default, a private SLB instance will be created for the API Server, and users can modify the SLB instance specifications. Deleting this instance will result in the API Server being unable to access.
      zh-cn: 默认将为 API Server 创建一个私网 SLB 实例，用户可修改SLB实例规格。若删除该实例会导致 API Server 无法访问。
    AssociationPropertyMetadata:
      ZoneId: ${ZoneId}
  ClusterInternetMaxBandwidthOut:
    Default: 100
    Type: Number
    Label:
      zh-cn: 集群公网带宽
      en: Master Node Internet Max Bandwidth Out
    MinValue: 1
    MaxValue: 100
    Description:
      zh-cn: 取值范围1-100
      en: The value range is from 1 to 100.
    AssociationPropertyMetadata:
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateSNAT}
            - true
  ClusterInternetChargeType:
    Default: PayByTraffic
    Type: String
    Label:
      zh-cn: NAT公网流量付费类型
      en: Internet Charge Type of NAT.
    AllowedValues:
      - PayByBandwidth
      - PayByTraffic
    AssociationPropertyMetadata:
      LocaleKey: InternetChargeType
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateSNAT}
            - true
  ApiServerInternetMaxBandwidthOut:
    Default: 100
    Type: Number
    Label:
      zh-cn: ApiServer公网带宽
      en: Master Node Internet Max Bandwidth Out
    MinValue: 1
    MaxValue: 100
    Description:
      zh-cn: 取值范围1-100
      en: The value range is from 1 to 100.
    AssociationPropertyMetadata:
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateApiServerEip}
            - true
  ApiServerInternetChargeType:
    Default: PayByTraffic
    Type: String
    Label:
      zh-cn: ApiServer公网流量付费类型
      en: Internet Charge Type of NAT.
    AllowedValues:
      - PayByBandwidth
      - PayByTraffic
    AssociationPropertyMetadata:
      LocaleKey: InternetChargeType
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateApiServerEip}
            - true
  MasterEcsInstanceType:
    Type: String
    Label:
      en: Instance Type
      zh-cn: 实例类型
    Description:
      en: Master nodes must have a minimum of 2 cores and 4GB of RAM.
      zh-cn: 实例类型，Master节点不可小于2核4G，否则可能会创建失败。
    AssociationProperty: ALIYUN::ECS::Instance::InstanceType
    AssociationPropertyMetadata:
      ZoneId: ${ZoneId}
      InstanceChargeType: ${InstanceChargeType}
  MasterAmount:
    Type: Number
    Default: 1
    Label:
      en: Master amount.
      zh-cn: Master节点数量
    Description:
      zh-cn: Master节点数量，最少为1。
      en: Master amount, must at least 1.
    MinValue: 1
  MasterSystemDiskCategory:
    Default: cloud_essd
    Label:
      zh-cn: 系统盘类型
      en: System Disk Type
    Type: String
    Description:
      zh-cn: 系统盘类型，须选择实例支持的磁盘类型。
      en: System disk type, you must select the type of disk that the instance.
    AllowedValues:
      - cloud_efficiency
      - cloud_ssd
      - cloud_essd
    AssociationPropertyMetadata:
      LocaleKey: DiskCategory
  MasterSystemDiskSize:
    Default: 40
    Type: Number
    Description:
      zh-cn: 系统盘大小, 取值范围：[40, 500], 单位：GB。
      en: 'System disk size of each node, range of values: 40-500, units: GB.'
    MinValue: 40
    MaxValue: 500
    Label:
      zh-cn: 系统盘空间
      en: System Disk Space
  MasterDataDiskCategory:
    Default: cloud_essd
    Label:
      zh-cn: 数据盘类型
      en: Data Disk Type
    Type: String
    Description:
      zh-cn: 数据盘类型，须选择实例支持的磁盘类型。
      en: Data disk type, you must select the type of disk that the instance.
    AllowedValues:
      - cloud_efficiency
      - cloud_ssd
      - cloud_essd
    AssociationPropertyMetadata:
      LocaleKey: DiskCategory
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateMasterDataDisk}
            - true
  MasterDataDiskSize:
    Default: 40
    Type: Number
    Description:
      zh-cn: 数据盘大小, 取值范围：[40, 500], 单位：GB。
      en: 'System disk size of each node, range of values: 40-500, units: GB.'
    MinValue: 40
    MaxValue: 500
    Label:
      zh-cn: 数据盘空间
      en: Data Disk Space
    AssociationPropertyMetadata:
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateMasterDataDisk}
            - true
  MasterInternetMaxBandwidthOut:
    Default: 100
    Type: Number
    Label:
      zh-cn: Master节点公网带宽
      en: Master Node Internet Max Bandwidth Out
    MinValue: 1
    MaxValue: 100
    Description:
      zh-cn: 取值范围1-100
      en: The value range is from 1 to 100.
    AssociationPropertyMetadata:
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateMastersEip}
            - true
  MasterInternetChargeType:
    Default: PayByTraffic
    Type: String
    Label:
      zh-cn: 流量付费类型
      en: Internet Charge Type
    AllowedValues:
      - PayByBandwidth
      - PayByTraffic
    AssociationPropertyMetadata:
      LocaleKey: InternetChargeType
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateMastersEip}
            - true
  WorkerEcsInstanceType:
    Type: String
    Label:
      en: Instance Type
      zh-cn: 实例类型
    Description:
      en: Worker nodes must have a minimum of 1 core and 2GB of RAM.
      zh-cn: 实例类型，Worker节点不可小于1核2G，否则可能会创建失败。
    AssociationProperty: ALIYUN::ECS::Instance::InstanceType
    AssociationPropertyMetadata:
      ZoneId: ${ZoneId}
      InstanceChargeType: ${InstanceChargeType}
  WorkerInternetMaxBandwidthOut:
    Default: 100
    Type: Number
    Label:
      zh-cn: Worker节点公网带宽
      en: Master Node Internet Max Bandwidth Out
    MinValue: 1
    MaxValue: 100
    Description:
      zh-cn: 取值范围1-100
      en: The value range is from 1 to 100.
    AssociationPropertyMetadata:
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateWorkersEip}
            - true
  WorkerInternetChargeType:
    Default: PayByTraffic
    Type: String
    Label:
      zh-cn: 流量付费类型
      en: Internet Charge Type
    AllowedValues:
      - PayByBandwidth
      - PayByTraffic
    AssociationPropertyMetadata:
      LocaleKey: InternetChargeType
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateWorkersEip}
            - true
  WorkerAmount:
    Type: Number
    Default: 1
    Label:
      en: Worker amount.
      zh-cn: Worker节点数量
    Description:
      zh-cn: Worker节点数量，最少为1
      en: Worker amount, must at least 1.
    MinValue: 1
  WorkerSystemDiskCategory:
    Default: cloud_essd
    Label:
      zh-cn: 系统盘类型
      en: System Disk Type
    Type: String
    Description:
      zh-cn: 系统盘类型，须选择实例支持的磁盘类型。
      en: System disk type, you must select the type of disk that the instance.
    AllowedValues:
      - cloud_efficiency
      - cloud_ssd
      - cloud_essd
    AssociationPropertyMetadata:
      LocaleKey: DiskCategory
  WorkerSystemDiskSize:
    Default: 40
    Type: Number
    Description:
      zh-cn: 系统盘大小, 取值范围：[40, 500], 单位：GB。
      en: 'System disk size of each node, range of values: 40-500, units: GB.'
    MinValue: 40
    MaxValue: 500
    Label:
      zh-cn: 系统盘空间
      en: System Disk Space
  WorkerDataDiskCategory:
    Default: cloud_essd
    Label:
      zh-cn: 数据盘类型
      en: Data Disk Type
    Type: String
    Description:
      zh-cn: 数据盘类型，须选择实例支持的磁盘类型。
      en: Data disk type, you must select the type of disk that the instance.
    AllowedValues:
      - cloud_efficiency
      - cloud_ssd
      - cloud_essd
    AssociationPropertyMetadata:
      LocaleKey: DiskCategory
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateWorkerDataDisk}
            - true
  WorkerDataDiskSize:
    Default: 40
    Type: Number
    Description:
      zh-cn: 数据盘大小, 取值范围：[40, 500], 单位：GB。
      en: 'System disk size of each node, range of values: 40-500, units: GB.'
    MinValue: 40
    MaxValue: 500
    Label:
      zh-cn: 数据盘空间
      en: Data Disk Space
    AssociationPropertyMetadata:
      Visible:
        Condition:
          Fn::Equals:
            - ${NeedCreateWorkerDataDisk}
            - true
  InstancePassword:
    NoEcho: true
    Type: String
    Description:
      en: Server login password, Length 8-30, must contain three(Capital letters, lowercase letters, numbers, ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ Special symbol in)
      zh-cn: Master节点与Worker节点登录密码，长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号）。
    AllowedPattern: '[0-9A-Za-z\_\-\&:;''<>,=%`~!@#\(\)\$\^\*\+\|\{\}\[\]\.\?\/]+$'
    Label:
      en: Instance Password
      zh-cn: 实例密码
    ConstraintDescription:
      en: Length 8-30, must contain three(Capital letters, lowercase letters, numbers, ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ Special symbol in)
      zh-cn: 长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号）。
    MinLength: 8
    MaxLength: 30
    AssociationProperty: ALIYUN::ECS::Instance::Password
  AckDistroVersion:
    Type: String
    Label:
      zh-cn: Ack-Distro基础镜像版本
      en: Ack-Distro base image version.
    Description:
      zh-cn: Ack-Distro镜像版本，<a href="https://github.com/AliyunContainerService/ackdistro/releases/">版本说明</a>。
      en: Ack-Distro Image Version,<a href="https://github.com/AliyunContainerService/ackdistro/releases/">Version Description</a>.
    AllowedValues:
      - v1-22-15-ack-10
      - v1-20-11-ack-23
    Default: v1-22-15-ack-10
  NeedCreateApiServerEip:
    Type: Boolean
    Default: false
    Description:
      zh-cn: 创建Eip，并将该EIP与内网SLB实例绑定，获得从公网访问集群 API Server 的能力。
      en: Create an EIP and bind it to the internal SLB instance to gain the ability to access the API Server of the cluster from the internet.
    Label:
      zh-cn: 创建EIP用于API Server
      en: Create EIP for API Server.
  NeedCreateSNAT:
    Type: Boolean
    Default: false
    Description:
      zh-cn: >-
        为专有网络配置SNAT，这样集群节点和应用将具有公网访问能力。<br>
        若您所选的VPC中已有绑定了弹性公网的NAT网关，ACK将默认使用该网关。
      en: >-
        Configure SNAT for the proprietary network so that cluster nodes and applications have public network access.<br>
        If there is no NAT gateway or it is not bound to the elastic public network in your selected VPC,
        this option must be enabled, otherwise the creation will fail.<br>
        If a NAT gateway with an elastic public network bound to it already exists in your selected VPC, 
        you can leave this option unchecked and ACK will use the existing gateway by default.
    Label:
      zh-cn: 配置SNAT
      en: Create SNAT.
  NeedCreateMastersEip:
    Type: Boolean
    Default: false
    Label:
      zh-cn: 开启master节点公网
      en: Enable public network for master node.
  NeedCreateWorkersEip:
    Type: Boolean
    Default: false
    Label:
      zh-cn: 开启worker节点公网
      en: Enable public network for worker node.
  NeedCreateMasterDataDisk:
    Type: Boolean
    Default: false
    Label:
      zh-cn: 创建master节点数据盘
      en: Create master node data disk.
  NeedCreateWorkerDataDisk:
    Type: Boolean
    Default: false
    Label:
      zh-cn: 创建worker节点数据盘
      en: Create worker node data disk.
Conditions:
  IfCreateApiServerEIP:
    Fn::Equals:
      - Ref: NeedCreateApiServerEip
      - true
  IfCreateSNAT:
    Fn::Equals:
      - Ref: NeedCreateSNAT
      - true
  IfCreateMastersEip:
    Fn::Equals:
      - Ref: NeedCreateMastersEip
      - true
  IfCreateWorkersEip:
    Fn::Equals:
      - Ref: NeedCreateWorkersEip
      - true
  IfCreateMasterDataDisk:
    Fn::Equals:
      - Ref: NeedCreateMasterDataDisk
      - true
  IfCreateWorkerDataDisk:
    Fn::Equals:
      - Ref: NeedCreateWorkerDataDisk
      - true
Resources:
  # 新建安全组
  EcsSecurityGroup:
    Type: ALIYUN::ECS::SecurityGroup
    Properties:
      SecurityGroupName:
        Ref: ALIYUN::StackName
      VpcId:
        Ref: VpcId
      SecurityGroupEgress:
        - PortRange: '-1/-1'
          Priority: 1
          IpProtocol: all
          DestCidrIp: 0.0.0.0/0
          NicType: intranet
      # 仅开放api server端口
      SecurityGroupIngress:
        - PortRange: 6443/6443
          Priority: 1
          SourceCidrIp: 0.0.0.0/0
          IpProtocol: tcp
          NicType: intranet
  # 定义waitCondition和waitConditionHandle来等待命令执行完毕部署成功
  WaitCondition:
    Type: ALIYUN::ROS::WaitCondition
    Properties:
      Count: 1
      Handle:
        Ref: WaitConditionHandle
      Timeout: 2000
  WaitConditionHandle:
    Type: ALIYUN::ROS::WaitConditionHandle
  MasterEcsInstanceGroup:
    Type: ALIYUN::ECS::InstanceGroup
    Properties:
      ZoneId:
        Ref: ZoneId
      # 实例名
      InstanceName:
        Fn::Join:
          - '-'
          - - Ref: ALIYUN::StackName
            - 'master-[1,3]'
      IoOptimized: optimized
      # 付费类型
      InstanceChargeType:
        Ref: PayType
      PeriodUnit:
        Ref: PayPeriodUnit
      Period:
        Ref: PayPeriod
      # 网络配置
      VpcId:
        Ref: VpcId
      VSwitchId:
        Ref: VSwitchId
      SecurityGroupId:
        Ref: EcsSecurityGroup
      # 磁盘类型和大小
      SystemDiskCategory:
        Ref: MasterSystemDiskCategory
      SystemDiskSize:
        Ref: MasterSystemDiskSize
      DiskMappings:
        Fn::If:
          - IfCreateMasterDataDisk
          - - Category:
                Ref: MasterDataDiskCategory
              Size:
                Ref: MasterDataDiskSize
          - Ref: ALIYUN::NoValue
      MaxAmount:
        Ref: MasterAmount
      # 镜像
      ImageId: centos_7
      # 实例类型
      InstanceType:
        Ref: MasterEcsInstanceType
      Password:
        Ref: InstancePassword
      AllocatePublicIP:
        Ref: NeedCreateMastersEip
      InternetMaxBandwidthOut:
        Fn::If:
          - IfCreateMastersEip
          - Ref: MasterInternetMaxBandwidthOut
          - Ref: ALIYUN::NoValue
      InternetChargeType:
        Fn::If:
          - IfCreateMastersEip
          - Ref: MasterInternetChargeType
          - Ref: ALIYUN::NoValue
  Slb:
    Type: ALIYUN::SLB::LoadBalancer
    Properties:
      VpcId:
        Ref: VpcId
      VSwitchId:
        Ref: VSwitchId
      LoadBalancerName:
        Fn::Join:
          - '-'
          - - 'slb'
            - Ref: ALIYUN::StackName
      PayType:
        Ref: PayType
      PricingCycle:
        Ref: PayPeriodUnit
      Duration:
        Ref: PayPeriod
      AddressType: intranet
      LoadBalancerSpec:
        Ref: LoadBalancerSpec
      AutoPay: true
  SlbBackendServerAttachment:
    DependsOn:
      - MasterEcsInstanceGroup
    Type: ALIYUN::SLB::BackendServerAttachment
    Properties:
      BackendServerList:
        Fn::GetAtt:
          - MasterEcsInstanceGroup
          - InstanceIds
      LoadBalancerId:
        Ref: Slb
      BackendServerWeightList:
        - 100
        - 100
  SlbListener:
    DependsOn:
      - Slb
    Type: ALIYUN::SLB::Listener
    Properties:
      Persistence:
        CookieTimeout: 60
        StickySession: 'on'
        PersistenceTimeout: 180
        XForwardedFor: 'off'
        StickySessionType: insert
      ListenerPort: 6443
      Bandwidth: -1
      HealthCheck:
        HttpCode: http_2xx,http_3xx,http_4xx,http_5xx
        HealthCheckType: tcp
        UnhealthyThreshold: 3
        Timeout: 5
        HealthyThreshold: 3
        Port: 6443
        URI: /
        Interval: 2
      LoadBalancerId:
        Ref: Slb
      BackendServerPort: 6443
      Protocol: tcp
  EipSlbAddress:
    Type: ALIYUN::VPC::EIP
    Condition: IfCreateApiServerEIP
    Properties:
      Name:
        Ref: ALIYUN::StackName
      InternetChargeType:
        Ref: ApiServerInternetChargeType
      Bandwidth:
        Ref: ApiServerInternetMaxBandwidthOut
  EipSlbAddressAssociation:
    Type: ALIYUN::VPC::EIPAssociation
    Condition: IfCreateApiServerEIP
    Properties:
      InstanceId:
        Ref: Slb
      AllocationId:
        Ref: EipSlbAddress
  WorkerEcsInstanceGroup:
    Type: ALIYUN::ECS::InstanceGroup
    Properties:
      ZoneId:
        Ref: ZoneId
      # 实例名
      InstanceName:
        Fn::Join:
          - '-'
          - - Ref: ALIYUN::StackName
            - 'worker-[1,3]'
      IoOptimized: optimized
      # 付费类型
      InstanceChargeType:
        Ref: PayType
      PeriodUnit:
        Ref: PayPeriodUnit
      Period:
        Ref: PayPeriod
      # 网络配置
      VpcId:
        Ref: VpcId
      VSwitchId:
        Ref: VSwitchId
      SecurityGroupId:
        Ref: EcsSecurityGroup
      # 磁盘类型和大小
      SystemDiskCategory:
        Ref: WorkerSystemDiskCategory
      SystemDiskSize:
        Ref: WorkerSystemDiskSize
      DiskMappings:
        Fn::If:
          - IfCreateWorkerDataDisk
          - - Category:
                Ref: WorkerDataDiskCategory
              Size:
                Ref: WorkerDataDiskSize
          - Ref: ALIYUN::NoValue
      MaxAmount:
        Ref: WorkerAmount
      # 镜像
      ImageId: centos_7
      # 实例类型
      InstanceType:
        Ref: WorkerEcsInstanceType
      Password:
        Ref: InstancePassword
      AllocatePublicIP:
        Ref: NeedCreateWorkersEip
      InternetMaxBandwidthOut:
        Fn::If:
          - IfCreateWorkersEip
          - Ref: WorkerInternetMaxBandwidthOut
          - Ref: ALIYUN::NoValue
      InternetChargeType:
        Fn::If:
          - IfCreateWorkersEip
          - Ref: WorkerInternetChargeType
          - Ref: ALIYUN::NoValue
  InstallMonitor:
    Type: ALIYUN::ECS::RunCommand
    DependsOn:
      - WorkerEcsInstanceGroup
      - MasterEcsInstanceGroup
    Properties:
      CommandContent:
        Fn::Sub:
          - |
            #!/bin/sh
            ARGUS_VERSION=3.5.7 /bin/bash -c "$(curl -sS https://cms-agent-${RegionId}.oss-${RegionId}-internal.aliyuncs.com/Argus/agent_install_ecs-1.7.sh)" >> /root/install_cms_agent.log 2>&1
          - RegionId:
              Ref: ALIYUN::Region
      Type: RunShellScript
      InstanceIds:
          Fn::ListMerge:
            - Fn::GetAtt:
              - MasterEcsInstanceGroup
              - InstanceIds
            - Fn::GetAtt:
              - WorkerEcsInstanceGroup
              - InstanceIds
      Timeout: '200'
  # 到机器上执行命令
  InstanceRunCommand:
    Type: ALIYUN::ECS::RunCommand
    DependsOn:
      - WorkerEcsInstanceGroup
      - MasterEcsInstanceGroup
    Properties:
      CommandContent:
        Fn::Sub:
          - |
            #!/bin/bash
            # 安装sealer
            ARCH=amd64
            get_arch=`arch`
            if [[ $get_arch != "x86_64" ]];then
                ARCH=arm64
            fi
            # 部署物功能暂不支持参数化和特殊符合如中划线等，所以只能暂时这么写了
            if [[ ${!ARCH} == "amd64" ]];then
                # 根据版本选择对应的部署物
                echo "deploy amd64"
                # 使用-q是为了避免云助手执行时影响后续输出
                wget -q '{{ computenest::file::sealerAmd64 }}' -O sealer-latest-linux-amd64.tar.gz && \
                tar -xvf sealer-latest-linux-${!ARCH}.tar.gz -C /usr/bin
                if [[ ${AckDistroVersion} == "v1-22-15-ack-10" ]];then
                    wget -q '{{ computenest::file::ackdistroAmd64V12215Ack10 }}'
                elif [[ ${AckDistroVersion} == "v1-20-11-ack-23" ]];then
                    wget -q '{{ computenest::file::ackdistroAmd64V12011Ack23 }}'
                fi
            else
                echo "deploy arm64"
                wget -q '{{ computenest::file::sealerArm64 }}' -O sealer-latest-linux-arm64.tar.gz && \
                tar -xvf sealer-latest-linux-${!ARCH}.tar.gz -C /usr/bin
                if [[ ${AckDistroVersion} == "v1-22-15-ack-10" ]];then
                    wget -q '{{ computenest::file::ackdistroArm64V12215Ack10 }}'
                elif [[ ${AckDistroVersion} == "v1-20-11-ack-23" ]];then
                    wget -q '{{ computenest::file::ackdistroArm64V12011Ack23 }}'
                fi
            fi
            sealer load -i ackdistro-${!ARCH}-${AckDistroVersion}.tar
            sealer run ack-agility-registry.cn-shanghai.cr.aliyuncs.com/ecp_builder/ackdistro:${AckDistroVersion} -m ${Masters} -n ${Workers} -p ${Password} >> deployK8sOutput.txt 2>&1
            if [ ${NeedCreateApiServerEip} ];then
                # cert for external ip
                sed 's#https://apiserver.cluster.local:6443#https://${ExternalIp}:6443#g' ~/.kube/config > ~/.kube/external_ip_config
                sealer cert --alt-names ${ExternalIp}
                echo "公网方式 通过 kubectl 连接 Kubernetes 集群"
                echo "1. 安装和设置 kubectl 客户端。有关详细信息请参见: https://kubernetes.io/docs/tasks/tools/"
                echo "2. 配置集群凭证：将以下内容复制到本地计算机 $HOME/.kube/config 文件中（没有~/.kube目录请新建），配置完成后，即可使用 kubectl 从计算机访问 Kubernetes 集群。"
                cat  ~/.kube/external_ip_config
                echo "
  
                "
            fi
            sed 's#https://apiserver.cluster.local:6443#https://${InternalIp}:6443#g' ~/.kube/config > ~/.kube/internal_ip_config
            echo "私网方式 通过 kubectl 连接 Kubernetes 集群"
            echo "1. 安装和设置 kubectl 客户端。有关详细信息请参见: https://kubernetes.io/docs/tasks/tools/"
            echo "2. 配置集群凭证：将以下内容复制到本地计算机 $HOME/.kube/config 文件中（没有~/.kube目录请新建），配置完成后，即可使用 kubectl 从计算机访问 Kubernetes 集群。"
            cat  ~/.kube/internal_ip_config
            # 执行成功回调WaitCondition结束waitCondition的等待
            ${CurlCli} -d "{\"Data\" : \"Success\", \"status\" : \"SUCCESS\"}" > /dev/null 2>/dev/null
          - Masters:
              Fn::Join:
                - ','
                - Fn::GetAtt:
                    - MasterEcsInstanceGroup
                    - PrivateIps
            Workers:
              Fn::Join:
                - ','
                - Fn::GetAtt:
                    - WorkerEcsInstanceGroup
                    - PrivateIps
            Password:
              Ref: InstancePassword
            CurlCli:
              Fn::GetAtt:
                - WaitConditionHandle
                - CurlCli
            AckDistroVersion:
              Ref: AckDistroVersion
            NeedCreateApiServerEip:
              Ref: NeedCreateApiServerEip
            ExternalIp:
              Fn::If:
                - IfCreateApiServerEIP
                - Fn::GetAtt:
                    - EipSlbAddress
                    - EipAddress
                - Ref: ALIYUN::NoValue
            InternalIp:
              Fn::GetAtt:
                - Slb
                - IpAddress
      Type: RunShellScript
      InstanceIds:
        - Fn::Select:
            - 0
            - Fn::GetAtt:
                - MasterEcsInstanceGroup
                - InstanceIds
      Timeout: '2000'
  VpcNatGateway:
    Type: ALIYUN::VPC::NatGateway
    Condition: IfCreateSNAT
    Properties:
      NatGatewayName:
        Ref: ALIYUN::StackName
      VpcId:
        Ref: VpcId
      VSwitchId:
        Ref: VSwitchId
  # 创建用于公网访问的NAT网关的EIP
  ElasticIpForNat:
    Type: ALIYUN::VPC::EIP
    Condition: IfCreateSNAT
    Properties:
      Name:
        Ref: ALIYUN::StackName
      InternetChargeType:
        Ref: ClusterInternetChargeType
      Bandwidth:
        Ref: ClusterInternetMaxBandwidthOut
  # 绑定EIP和Nat网关
  ElasticIpForNatAssociation:
    Type: ALIYUN::VPC::EIPAssociation
    Condition: IfCreateSNAT
    DependsOn:
      - ElasticIpForNat
      - VpcNatGateway
    Properties:
      AllocationId:
        Fn::GetAtt:
          - ElasticIpForNat
          - AllocationId
      InstanceId:
        Fn::GetAtt:
          - VpcNatGateway
          - NatGatewayId
  # 配置NAT网关中的源地址转换表
  SNatEntry:
    Type: ALIYUN::ECS::SNatEntry
    Condition: IfCreateSNAT
    DependsOn:
      - ElasticIpForNatAssociation
    Properties:
      SNatIp:
        Fn::GetAtt:
          - ElasticIpForNat
          - EipAddress
      SNatTableId:
        Fn::GetAtt:
          - VpcNatGateway
          - SNatTableId
      SourceVSwitchId:
        Ref: VSwitchId
Outputs:
  ApiServerPublicAddress:
    Condition: IfCreateApiServerEIP
    Description:
      zh-cn: ApiServer公网地址
      en: Api Server Public IP Address
    Value:
      Fn::Sub:
        - ${ApiServerIp}:6443
        - ApiServerIp:
            Fn::GetAtt:
              - EipSlbAddress
              - EipAddress
  ApiServerPrivateAddress:
    Description:
      zh-cn: ApiServer私网地址
      en: Api Server Private IP Address
    Value:
      Fn::Sub:
        - ${ApiServerIp}:6443
        - ApiServerIp:
            Fn::GetAtt:
              - Slb
              - IpAddress
  NatEip:
    Condition: IfCreateSNAT
    Description:
      zh-cn: 用于集群公网访问的网关公网Ip
      en: Gateway public IP address used for public access to the cluster.
    Value:
      Fn::GetAtt:
        - ElasticIpForNat
        - EipAddress
  MasterPublicIp:
    Condition: IfCreateMastersEip
    Description:
      zh-cn: Master公网IP地址
      en: Master Private IP Addresses
    Value:
      Fn::GetAtt:
        - MasterEcsInstanceGroup
        - PublicIps
  MasterPrivateIp:
    Description:
      zh-cn: Master内网IP地址
      en: Master Private IP Addresses
    Value:
      Fn::GetAtt:
        - MasterEcsInstanceGroup
        - PrivateIps
  WorkerPublicIp:
    Condition: IfCreateWorkersEip
    Description:
      zh-cn: Worker公网IP地址
      en: Worker Private IP Addresses
    Value:
      Fn::GetAtt:
        - WorkerEcsInstanceGroup
        - PublicIps
  WorkerPrivateIp:
    Description:
      zh-cn: Worker内网IP地址
      en: Private IP Addresses
    Value:
      Fn::GetAtt:
        - WorkerEcsInstanceGroup
        - PrivateIps
  ConnectApiServer:
    Description:
      zh-cn: 如何连接ApiServer
      en: How Connect With Api Server
    Value:
      Fn::Sub:
        - |
          获取集群访问凭证方式: <br> 
          方式1：运维管理页中，选择执行"获取集群凭证"运维项，在执行结果中查看凭证。<br>
          方式2：远程连接master节点，公网访问凭证：~/.kube/external_ip_config，私网访问凭证： ~/.kube/internal_ip_config <br>
          方式3：<br>
          1.登录阿里云ECS控制台:${AssistantAddress}。<br>
          2.查看该命令执行Id: ${InvokeId} 的执行结果。
        - AssistantAddress:
            Fn::Sub:
              - |
                https://ecs.console.aliyun.com/cloud-assistant/region/${RegionId}/
              - RegionId:
                  Ref: ALIYUN::Region
          InvokeId:
            Fn::GetAtt:
              - InstanceRunCommand
              - InvokeId
Metadata:
  ALIYUN::ROS::Interface:
    # 分组信息
    ParameterGroups:
      - Parameters:
          - PayType
          - PayPeriodUnit
          - PayPeriod
        Label:
          default:
            en: PayType Configuration
            zh-cn: 付费类型配置
      - Parameters:
          - ZoneId
        Label:
          default:
            zh-cn: 可用区配置
            en: Zone Configuration
      - Parameters:
          - LoadBalancerSpec
        Label:
          default:
            zh-cn: 负载均衡配置
            en: Slb Configuration
      - Parameters:
          - AckDistroVersion
        Label:
          default:
            zh-cn: Ack-Distro镜像版本
            en: Ack-Distro image version
      - Parameters:
          - VpcId
          - VSwitchId
          - AutoCreateSecurityGroup
          - NeedCreateSNAT
          - ClusterInternetChargeType
          - ClusterInternetMaxBandwidthOut
          - NeedCreateApiServerEip
          - ApiServerInternetChargeType
          - ApiServerInternetMaxBandwidthOut
        Label:
          default:
            zh-cn: 网络配置
            en: Network Configuration
      - Parameters:
          - MasterAmount
          - MasterEcsInstanceType
          - MasterSystemDiskCategory
          - MasterSystemDiskSize
          - NeedCreateMasterDataDisk
          - MasterDataDiskCategory
          - MasterDataDiskSize
          - NeedCreateMastersEip
          - MasterInternetChargeType
          - MasterInternetMaxBandwidthOut
        Label:
          default:
            en: Instance Configuration
            zh-cn: Master实例配置
      - Parameters:
          - WorkerAmount
          - WorkerEcsInstanceType
          - WorkerSystemDiskCategory
          - WorkerSystemDiskSize
          - NeedCreateWorkerDataDisk
          - WorkerDataDiskCategory
          - WorkerDataDiskSize
          - NeedCreateWorkersEip
          - WorkerInternetChargeType
          - WorkerInternetMaxBandwidthOut
        Label:
          default:
            en: Instance Configuration
            zh-cn: Worker实例配置
      - Parameters:
          - InstancePassword
        Label:
          default:
            en: Instance Password Configuration
            zh-cn: 节点密码配置