deploy/pipelines/07-sap-cal-installation.yaml

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

---
# /*---------------------------------------------------------------------------8
# |                                                                            |
# |               This pipeline performs the software installation             |
# |              and must run on a self hosted deployment agent                |
# |                      due to long run time.                                 |
# |                                                                            |
# +------------------------------------4--------------------------------------*/

parameters:
  - name:                              sap_system_configuration_name
    displayName:                       "SAP System configuration name, use the following syntax: ENV-LOCA-VNET-SID"
    type:                              string
    default:                           DEV-WEEU-SAP01-X00

  - name:                              environment
    displayName:                       Workload Environment (DEV, QUA, PRD, ...)
    type:                              string
    default:                           DEV

  - name:                              sap_cal_product_name
    displayName:                       SAP CAL Product Name
    type:                              string

  - name:                              extra_params
    displayName:                       Extra Parameters
    type:                              string
    default:                           ""

  - name:                              base_os_configuration
    displayName:                       Core Operating System Configuration
    type:                              boolean
    default:                           true

  - name:                              sap_os_configuration
    displayName:                       SAP Operating System Configuration
    type:                              boolean
    default:                           true

  - name:                              sapcal_integration
    displayName:                       SAP CAL Integration
    type:                              boolean
    default:                           false

# 20220929 MKD - ACSS Registration <BEGIN>
  - name:                             acss_registration
    displayName:                      Register System in ACSS
    type:                             boolean
    default:                          true

  - name:                             acss_environment
    displayName:                      ACSS Prod/NonProd
    type:                             string
    values:
      - NonProd
      - Prod

  - name:                              acss_sap_product
    displayName:                       System Type
    type:                              string
    values:
      - S4HANA
      - ECC
      - Other
  # 20220929 MKD - ACSS Registration <END>

  - name:                              sap_automation_repo_path
    displayName:                       The local path on the agent where the sap_automation repo can be found
    type:                              string

  - name:                              config_repo_path
    displayName:                       The local path on the agent where the config repo can be found
    type:                              string

stages:
  - stage: Preparation_for_Ansible
    condition: and(not(failed()), not(canceled()))
    variables:
      - template:                      variables/07-sap-cal-installation-variables.yaml
        parameters:
          environment:                 ${{ parameters.environment }}
    displayName:                       OS Configuration and SAP Installation
    jobs:
      - job:                           Installation_step
        displayName:                   OS Configuration and SAP Installation
        timeoutInMinutes:              0
        workspace:
          clean:                       all
        steps:
          - template: templates\download.yaml
            parameters:
              getLatestFromBranch: true
          - task:                      PostBuildCleanup@4
          - bash: |
              #!/bin/bash
              # Exit immediately if a command exits with a non-zero status.
              set -e

              green="\e[1;32m" ; reset="\e[0m" ; boldred="\e[1;31m"
              if [[ ! -f /etc/profile.d/deploy_server.sh ]]; then
                echo -e "$green --- Install dos2unix ---$reset"
                  sudo apt-get -qq install dos2unix
              fi
              echo -e "$green--- Convert config file to UX format ---$reset"
              echo -e "$green--- Update .sap_deployment_automation/config as DEPLOYMENT_REPO_PATH can change on devops agent ---$reset"
                export HOME=${CONFIG_REPO_PATH}/$(Deployment_Configuration_Path)
                cd $HOME

              echo -e "$green--- Configure devops CLI extension ---$reset"
                az config set extension.use_dynamic_install=yes_without_prompt  --output none

                az extension add --name azure-devops --output none

                az devops configure --defaults organization=$(System.CollectionUri) project='$(System.TeamProject)' --output none

              echo -e "$green--- Validations ---$reset"
                ENVIRONMENT=$(echo ${SAP_SYSTEM_CONFIGURATION_NAME} | awk -F'-' '{print $1}' | xargs) ; echo Environment $ENVIRONMENT
                   LOCATION=$(echo ${SAP_SYSTEM_CONFIGURATION_NAME} | awk -F'-' '{print $2}' | xargs) ; echo Location    $LOCATION
                    NETWORK=$(echo ${SAP_SYSTEM_CONFIGURATION_NAME} | awk -F'-' '{print $3}' | xargs) ; echo Virtual network logical name $NETWORK
                        SID=$(echo ${SAP_SYSTEM_CONFIGURATION_NAME} | awk -F'-' '{print $4}' | xargs) ; echo SID $SID

                environment_file_name=$HOME/.sap_deployment_automation/$ENVIRONMENT$LOCATION$NETWORK ; echo configuration_file $environment_file_name
                          params_file=$HOME/SYSTEM/${SAP_SYSTEM_CONFIGURATION_NAME}/sap-parameters.yaml              ;  echo sap_parameters_file $params_file

              if [ "azure pipelines" = "$(this_agent)" ]; then
                  echo "##vso[task.logissue type=error]Please use a self hosted agent for this playbook. Define it in the SDAF-${ENVIRONMENT} variable group using the 'POOL' variable."
                  exit 2
                fi

                if [ ! -f $environment_file_name ]; then
                  echo -e "$boldred--- $environment_file_name was not found ---$reset"
                  echo "##vso[task.logissue type=error]Workload zone configuration file $environment_file_name was not found."
                  exit 2
                fi

                if [ ! -f $params_file ]; then
                  echo -e "$boldred--- $HOME/SYSTEM/${SAP_SYSTEM_CONFIGURATION_NAME}/sap-parameters.yaml was not found ---$reset"
                  echo "##vso[task.logissue type=error]File $HOME/SYSTEM/${SAP_SYSTEM_CONFIGURATION_NAME}/sap-parameters.yaml was not found."
                  exit 2
                else
                  dos2unix -q $HOME/SYSTEM/${SAP_SYSTEM_CONFIGURATION_NAME}/sap-parameters.yaml
                fi

                if [ ! -n ${SID} ]; then
                  echo "##vso[task.logissue type=error]SID was not found in ${SAP_SYSTEM_CONFIGURATION_NAME}."
                  exit 2
                fi

                if [ ! -f $HOME/SYSTEM/${SAP_SYSTEM_CONFIGURATION_NAME}/${SID}_hosts.yaml ]; then
                  echo -e "$boldred--- $HOME/SYSTEM/${SAP_SYSTEM_CONFIGURATION_NAME}/${SID}_hosts.yaml was not found ---$reset"
                  echo "##vso[task.logissue type=error]File $HOME/SYSTEM/${SAP_SYSTEM_CONFIGURATION_NAME}/${SID}_hosts.yaml was not found."
                  exit 2
                fi
                dos2unix -q $HOME/SYSTEM/${SAP_SYSTEM_CONFIGURATION_NAME}/${SID}_hosts.yaml

                export VARIABLE_GROUP_ID=$(az pipelines variable-group list --query "[?name=='$(variable_group)'].id | [0]")
                echo '$(variable_group) id: ' $VARIABLE_GROUP_ID
                if [ -z ${VARIABLE_GROUP_ID} ]; then
                    echo "##vso[task.logissue type=error]Variable group $(variable_group) could not be found."
                    exit 2
                fi

                echo "##vso[build.updatebuildnumber]Deploying ${SAP_SYSTEM_CONFIGURATION_NAME} using SAP CAL"

                echo "##vso[task.setvariable variable=SID;isOutput=true]${SID}"
                echo "##vso[task.setvariable variable=SAP_PARAMETERS;isOutput=true]sap-parameters.yaml"
                echo "##vso[task.setvariable variable=FOLDER;isOutput=true]$HOME/SYSTEM/${SAP_SYSTEM_CONFIGURATION_NAME}"
                echo "##vso[task.setvariable variable=HOSTS;isOutput=true]${SID}_hosts.yaml"

              echo -e "$green--- Get Files from the DevOps Repository ---$reset"
                cd ${CONFIG_REPO_PATH}/$(Deployment_Configuration_Path)/SYSTEM/${SAP_SYSTEM_CONFIGURATION_NAME}
                sap_params_updated=0

                fqdn="$(grep -m1 "$sap_fqdn:" sap-parameters.yaml | cut -d':' -f2-  | tr -d ' ' | tr -d '"')"
                if [ -z $fqdn ] ; then
                  sed -i 's|sap_fqdn:.*|sap_fqdn:                      '"$(sap_fqdn)"'|'  sap-parameters.yaml
                fi

                if [[ -n "${sapcalProductName}" ]]; then
                  echo -e "$green--- Add SAP CAL Product Name $sapcalProductName to sap-parameters.yaml ---$reset"
                  sed -i 's|sap_cal_product_name:.*|sap_cal_product_name:                    '"$sapcalProductName"'|'  sap-parameters.yaml
                fi

              echo -e "$green--- Get connection details ---$reset"
                mkdir -p artifacts

                az_var=$(az pipelines variable-group variable list --group-id ${VARIABLE_GROUP_ID} --query "${NETWORK}"Workload_Key_Vault.value --output tsv)
                if [ -z ${az_var} ]; then
                  export workload_key_vault=$(cat "${environment_file_name}"  | grep workloadkeyvault      | awk -F'=' '{print $2}' | xargs) ; echo 'Workload Key Vault' ${workload_key_vault}
                else
                  export workload_key_vault=${az_var} ; echo 'Workload Key Vault' ${workload_key_vault} ;
                fi

                az_var=$(az pipelines variable-group variable list --group-id ${VARIABLE_GROUP_ID} --query "${NETWORK}"Workload_Secret_Prefix.value  --output tsv)
                if [ -z ${az_var} ]; then
                  export workload_prefix=$(cat "${environment_file_name}"  | grep workload_zone_prefix      | awk -F'=' '{print $2}' | xargs) ; echo 'Workload Prefix' ${workload_prefix}
                else
                  export workload_prefix=${az_var} ; echo 'Workload Prefix' ${workload_prefix};
                fi

                az_var=$(az pipelines variable-group variable list --group-id ${VARIABLE_GROUP_ID} --query Terraform_Remote_Storage_Subscription.value  --output tsv)
                if [ -z ${az_var} ]; then
                  export control_plane_subscription=$(cat "${environment_file_name}"  | grep STATE_SUBSCRIPTION      | awk -F'=' '{print $2}' | xargs) ; echo 'Control Plane Subscription' ${control_plane_subscription}
                else
                  export control_plane_subscription=${az_var} ; echo 'Control Plane Subscription' ${control_plane_subscription}
                fi

                if [[ $EXTRA_PARAMETERS = "'$(EXTRA_PARAMETERS)'" ]]; then
                  new_parameters=$PIPELINE_EXTRA_PARAMETERS
                else
                  echo "##vso[task.logissue type=warning]Extra parameters were provided - '$EXTRA_PARAMETERS'"
                  new_parameters="$EXTRA_PARAMETERS $PIPELINE_EXTRA_PARAMETERS"
                fi

                echo "##vso[task.setvariable variable=SSH_KEY_NAME;isOutput=true]${workload_prefix}-sid-sshkey"
                echo "##vso[task.setvariable variable=VAULT_NAME;isOutput=true]$workload_key_vault"
                echo "##vso[task.setvariable variable=PASSWORD_KEY_NAME;isOutput=true]${workload_prefix}-sid-password"
                echo "##vso[task.setvariable variable=USERNAME_KEY_NAME;isOutput=true]${workload_prefix}-sid-username"
                echo "##vso[task.setvariable variable=NEW_PARAMETERS;isOutput=true]${new_parameters}"
                echo "##vso[task.setvariable variable=CP_SUBSCRIPTION;isOutput=true]${control_plane_subscription}"


              echo -e "$green--- az login ---$reset"
                # If the deployer_file exists we run on a deployer configured by the framework instead of a azdo hosted one
                deployer_file=/etc/profile.d/deploy_server.sh
                if [ "$USE_MSI" = "true" ]; then
                  echo  "Using MSI"
                  source /etc/profile.d/deploy_server.sh
                  az account set --subscription $control_plane_subscription

                else
                  if [ ! -n $AZURE_CLIENT_ID ]; then
                    echo "##vso[task.logissue type=error]Variable ARM_CLIENT_ID was not defined."
                    exit 2
                  fi

                  if [ ! -n $AZURE_CLIENT_SECRET ]; then
                    echo "##vso[task.logissue type=error]Variable ARM_CLIENT_SECRET was not defined."
                    exit 2
                  fi

                  if [ ! -n $AZURE_TENANT_ID ]; then
                    echo "##vso[task.logissue type=error]Variable ARM_TENANT_ID was not defined."
                    exit 2
                  fi
                  az login --service-principal --username $AZURE_CLIENT_ID --password=${AZURE_CLIENT_SECRET} --tenant $AZURE_TENANT_ID --output none
                  return_code=$?
                  if [ 0 != $return_code ]; then
                    echo -e "$boldred--- Login failed ---$reset"
                    echo "##vso[task.logissue type=error]az login failed."
                    exit $return_code
                  fi
                  az account set --subscription $control_plane_subscription
                fi

                return_code=$?
                if [ 0 != $return_code ]; then
                  echo -e "$boldred--- Login failed ---$reset"
                  echo "##vso[task.logissue type=error]az login failed."
                  exit $return_code
                fi

                az_var=$(az pipelines variable-group variable list --group-id ${VARIABLE_GROUP_ID} --query "Deployer_Key_Vault.value" | tr -d \")
                if [ -n ${az_var} ]; then
                    kv_name=${az_var}; echo "Key Vault="$kv_name
                else
                    kv_name=$(cat .sap_deployment_automation/$(environment_code)$(location_code) | grep keyvault |awk -F'=' '{print $2}'); echo "Key Vault="$kv_name
                fi

                echo "##vso[task.setvariable variable=KV_NAME;isOutput=true]$kv_name"

                if [ "your S User" == "${SUsername}" ]; then
                  echo "##vso[task.logissue type=error]Please define the S-Username variable."
                  exit 2
                fi

                if [ "your S user password" == "${SPassword}" ]; then
                  echo "##vso[task.logissue type=error]Please define the S-Password variable."
                  exit 2
                fi

              echo -e "$green--- Set S-Username and S-Password in the key_vault if not yet there ---$reset"

                export SUsernamefromVault=$(az keyvault secret list --vault-name "${kv_name}" --subscription "${ARM_SUBSCRIPTION_ID}" --query "[].{Name:name} | [? contains(Name,'S-Username')] | [0]"  -o tsv)
                if [ $SUsernamefromVault == $SUsername ]; then
                  echo -e "$green--- $SUsername present in keyvault. In case of download errors check that user and password are correct ---$reset"
                  echo "##vso[task.setvariable variable=SUSERNAME;isOutput=true]$SUsernamefromVault"
                else
                  echo -e "$green--- Setting the S username in key vault ---$reset"
                  az keyvault secret set --name "S-Username" --vault-name $kv_name --value="${SUsername}" --subscription "${ARM_SUBSCRIPTION_ID}" --expires "$(date -d '+1 year' -u +%Y-%m-%dT%H:%M:%SZ)" --output none
                  echo "##vso[task.setvariable variable=SUSERNAME;isOutput=true]$SUsername"
                fi

                export SPasswordfromVault=$(az keyvault secret list --vault-name "${kv_name}" --subscription "${ARM_SUBSCRIPTION_ID}" --query "[].{Name:name} | [? contains(Name,'S-Password')] | [0]"  -o tsv)
                if [ ${SPassword} == $SPasswordfromVault ]; then
                  echo "##vso[task.setvariable variable=SPASSWORD;isOutput=true]${SPasswordfromVault}"
                  echo -e "$green--- Password present in keyvault. In case of download errors check that user and password are correct ---$reset"
                else
                  echo -e "$green--- Setting the S user name password in key vault ---$reset"
                  az keyvault secret set --name "S-Password" --vault-name $kv_name --value "${SPassword}" --subscription "${ARM_SUBSCRIPTION_ID}" --expires "$(date -d '+1 year' -u +%Y-%m-%dT%H:%M:%SZ)" --output none
                  echo "##vso[task.setvariable variable=SPASSWORD;isOutput=true]${SPassword}"
                fi

                az keyvault secret show --name ${workload_prefix}-sid-sshkey --vault-name $workload_key_vault --subscription $AZURE_SUBSCRIPTION_ID --query value -o tsv > artifacts/${SAP_SYSTEM_CONFIGURATION_NAME}_sshkey
                cp sap-parameters.yaml artifacts/.
                cp ${SID}_hosts.yaml artifacts/.

                2> >(while read line; do (>&2 echo "STDERROR: $line"); done)
            name:                            Preparation
            displayName:                     Preparation for Ansible
            env:
              SCRIPT_PATH:                   $${{ parameters.sap_automation_repo_path }}/deploy/pipelines/templates/*.sh
              SYSTEM_ACCESSTOKEN:            $(System.AccessToken)
              AZURE_DEVOPS_EXT_PAT:          $(System.AccessToken)
              ANSIBLE_HOST_KEY_CHECKING:     false
              AZURE_CLIENT_ID:               $(ARM_CLIENT_ID)
              AZURE_CLIENT_SECRET:           $(ARM_CLIENT_SECRET)
              AZURE_TENANT_ID:               $(ARM_TENANT_ID)
              AZURE_SUBSCRIPTION_ID:         $(Terraform_Remote_Storage_Subscription)
              ANSIBLE_COLLECTIONS_PATH:      /opt/ansible/collections
              CONFIG_REPO_PATH:              ${{ parameters.config_repo_path }}
              SAP_SYSTEM_CONFIGURATION_NAME: ${{ parameters.sap_system_configuration_name }}
              EXTRA_PARAMETERS:              $(EXTRA_PARAMETERS)
              PIPELINE_EXTRA_PARAMETERS:     ${{ parameters.extra_params }}
              USE_MSI:                       $(USE_MSI)
              SUsername:                     $(S-Username)
              SPassword:                     $(S-Password)
              sapcalProductName:             ${{ parameters.sap_cal_product_name }}

          - template:                        templates\run-ansible.yaml
            parameters:
              displayName:                   "Parameter validation"
              ansibleFilePath:               ${{ parameters.sap_automation_repo_path }}/deploy/ansible/playbook_00_validate_parameters.yaml
              secretName:                    "$(Preparation.SSH_KEY_NAME)"
              passwordSecretName:            "$(Preparation.PASSWORD_KEY_NAME)"
              userNameSecretName:            "$(Preparation.USERNAME_KEY_NAME)"
              vaultName:                     $(Preparation.VAULT_NAME)
              parametersFolder:              $(Preparation.FOLDER)
              sapParams:                     "${{ parameters.config_repo_path }}/$(Deployment_Configuration_Path)/SYSTEM/${{ parameters.sap_system_configuration_name }}/artifacts/$(Preparation.SAP_PARAMETERS)"
              sidHosts:                      $(Preparation.HOSTS)
              extraParams:                   "$(Preparation.NEW_PARAMETERS)"
              azureClientId:                 $(ARM_CLIENT_ID)
              azureClientSecret:             $(ARM_CLIENT_SECRET)
              azureTenantId:                 $(ARM_TENANT_ID)
              azureSubscriptionId:           $(ARM_SUBSCRIPTION_ID)
              USE_MSI:                       $(USE_MSI)
          - ${{ if eq(parameters.base_os_configuration, true) }}:
              - template:                    templates\run-ansible.yaml
                parameters:
                  displayName:               "Operating System Configuration"
                  ansibleFilePath:           ${{ parameters.sap_automation_repo_path }}/deploy/ansible/playbook_01_os_base_config.yaml
                  secretName:                "$(Preparation.SSH_KEY_NAME)"
                  passwordSecretName:        "$(Preparation.PASSWORD_KEY_NAME)"
                  userNameSecretName:        "$(Preparation.USERNAME_KEY_NAME)"
                  vaultName:                 $(Preparation.VAULT_NAME)
                  parametersFolder:          $(Preparation.FOLDER)
                  sapParams:                 "${{ parameters.config_repo_path }}/$(Deployment_Configuration_Path)/SYSTEM/${{ parameters.sap_system_configuration_name }}/artifacts/$(Preparation.SAP_PARAMETERS)"
                  sidHosts:                  $(Preparation.HOSTS)
                  extraParams:               "$(Preparation.NEW_PARAMETERS)"
                  azureClientId:             $(ARM_CLIENT_ID)
                  azureClientSecret:         $(ARM_CLIENT_SECRET)
                  azureTenantId:             $(ARM_TENANT_ID)
                  azureSubscriptionId:       $(ARM_SUBSCRIPTION_ID)
                  USE_MSI:                   $(USE_MSI)
          - ${{ if eq(parameters.sap_os_configuration, true) }}:
              - template:                    templates\run-ansible.yaml
                parameters:
                  displayName:               "SAP Specific Operating System Configuration"
                  ansibleFilePath:           ${{ parameters.sap_automation_repo_path }}/deploy/ansible/playbook_02_os_sap_specific_config.yaml
                  secretName:                "$(Preparation.SSH_KEY_NAME)"
                  passwordSecretName:        "$(Preparation.PASSWORD_KEY_NAME)"
                  userNameSecretName:        "$(Preparation.USERNAME_KEY_NAME)"
                  vaultName:                 $(Preparation.VAULT_NAME)
                  parametersFolder:          $(Preparation.FOLDER)
                  sapParams:                 "${{ parameters.config_repo_path }}/$(Deployment_Configuration_Path)/SYSTEM/${{ parameters.sap_system_configuration_name }}/artifacts/$(Preparation.SAP_PARAMETERS)"
                  sidHosts:                  $(Preparation.HOSTS)
                  extraParams:               "$(Preparation.NEW_PARAMETERS)"
                  azureClientId:             $(ARM_CLIENT_ID)
                  azureClientSecret:         $(ARM_CLIENT_SECRET)
                  azureTenantId:             $(ARM_TENANT_ID)
                  azureSubscriptionId:       $(ARM_SUBSCRIPTION_ID)
                  USE_MSI:                   $(USE_MSI)
          - ${{ if eq(parameters.sapcal_integration, true) }}:
              - template:                    templates\run-ansible.yaml
                parameters:
                  displayName:               "SAPCAL Integration"
                  ansibleFilePath:           ${{ parameters.sap_automation_repo_path }}/deploy/ansible/playbook_sapcal_integration.yaml
                  secretName:                "$(Preparation.SSH_KEY_NAME)"
                  passwordSecretName:        "$(Preparation.PASSWORD_KEY_NAME)"
                  userNameSecretName:        "$(Preparation.USERNAME_KEY_NAME)"
                  vaultName:                 $(Preparation.VAULT_NAME)
                  parametersFolder:          $(Preparation.FOLDER)
                  sapParams:                 "${{ parameters.config_repo_path }}/$(Deployment_Configuration_Path)/SYSTEM/${{ parameters.sap_system_configuration_name }}/artifacts/$(Preparation.SAP_PARAMETERS)"
                  sidHosts:                  $(Preparation.HOSTS)
                  extraParams:               "$(Preparation.NEW_PARAMETERS)"
                  azureClientId:             $(ARM_CLIENT_ID)
                  azureClientSecret:         $(ARM_CLIENT_SECRET)
                  azureTenantId:             $(ARM_TENANT_ID)
                  azureSubscriptionId:       $(ARM_SUBSCRIPTION_ID)
                  sapcalProductName:         ${{ parameters.sap_cal_product_name }}
                  USE_MSI:                   $(USE_MSI)
          - template:                        templates\collect-calapi-file.yaml
            parameters:
              filePath:                      ${{ parameters.config_repo_path }}/$(Deployment_Configuration_Path)/SYSTEM/${{ parameters.sap_system_configuration_name }}