vsts/windowsLinuxAndAndroidBuildMatrixConfig.yaml

name: $(BuildID)_$(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)

trigger:
  batch: true
  branches:
    include:
      - main
  paths:
    exclude:
      - doc/*
      - device/iot-device-samples/*
      - service/iot-service-samples/*
      - provisioning/provisioning-samples/*

variables:
  # Run JDK 8 + 11 in parallel if it is a pull request, but in serial otherwise
  ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
    maxParallel: 2
  ${{ else }}:
    maxParallel: 1

jobs:
  ### Windows ###
  - job: Windows
    timeoutInMinutes: 180
    strategy:
      maxParallel: $[variables.maxParallel]
      matrix:
        JDK 11: # always run Java 11
          JAVA_VERSION: 11
        JDK 8: # only run Java 8 in nightly or CI builds.
          JAVA_VERSION: 8
    pool:
      vmImage: windows-latest
    displayName: Windows
    steps:
      - powershell: ./vsts/echo_versions.ps1
        displayName: 'Echo Versions'
        env:
          JAVA_VERSION: $(JAVA_VERSION)
          COMMIT_FROM: $(COMMIT_FROM)

      - powershell: ./vsts/start_tpm_windows.ps1
        displayName: 'Start TPM Simulator'
        env:
          COMMIT_FROM: $(COMMIT_FROM)

      - powershell: ./vsts/build_repo.ps1
        displayName: 'Build and Test'
        env:
          JAVA_VERSION: $(JAVA_VERSION)
          IOT_DPS_CONNECTION_STRING: $(IOT-DPS-CONNECTION-STRING)
          IOT_DPS_ID_SCOPE: $(IOT-DPS-ID-SCOPE)
          IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
          DPS_GLOBALDEVICEENDPOINT_INVALIDCERT: $(DPS-GLOBALDEVICEENDPOINT-INVALIDCERT)
          PROVISIONING_CONNECTION_STRING_INVALIDCERT: $(PROVISIONING-CONNECTION-STRING-INVALIDCERT)
          IS_BASIC_TIER_HUB: $(IS-BASIC-TIER-HUB)
          TARGET_BRANCH: $(System.PullRequest.TargetBranch)
          RECYCLE_TEST_IDENTITIES: true

      - task: CopyFiles@2
        displayName: 'Copy Test Results to Artifact Staging Directory'
        inputs:
          SourceFolder: '$(Build.SourcesDirectory)'
          Contents: |
            **/*.trx
            **/*.xml
          TargetFolder: '$(Build.ArtifactStagingDirectory)'
        continueOnError: true

      - task: CopyFiles@2
        displayName: 'Copy Build Results to Artifact Staging Directory'
        condition: always()
        inputs:
          flattenFolders: true
          SourceFolder: '$(Build.SourcesDirectory)'
          Contents: |
            **/target/*.jar
            **/target/*.pom
            !**/target/*-sources.jar
            !**/target/*-javadoc.jar
            !**/target/dice-*
            !**/target/provisioning-x509-cert-generator*
            !**/target/*emulator*
            !**/*sample*/**/target/**
            !**/*test*/**/target/**
          TargetFolder: '$(Build.ArtifactStagingDirectory)/buildOutput'
        continueOnError: true

      - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
        displayName: 'Generate SBOM for Build Artifacts'
        condition: always()
        inputs:
          BuildDropPath: '$(Build.ArtifactStagingDirectory)/buildOutput'

      - task: PublishBuildArtifacts@1
        displayName: 'Publish Artifact Staging Directory'
        condition: always()
        continueOnError: true

      - task: PublishTestResults@2
        displayName: 'Publish Test Results'
        condition: always()
        inputs:
          mergeTestResults: true
          testRunTitle: "Windows JDK $(JAVA_VERSION) (Attempt $(System.JobAttempt))"
        continueOnError: true

      - task: ComponentGovernanceComponentDetection@0
        displayName: Component Governance Detection
        condition: always()
        inputs:
          scanType: 'Register'
          verbosity: 'Verbose'
          alertWarningLevel: 'Low' # The task will present a warning, but will not cause the build to fail

  ### Linux ###
  - job: Linux
    timeoutInMinutes: 180
    pool:
      # If this is changed, don't forget to update supported_platforms.md in the root directory. That document outlines what OS we test on and should stay up to date.
      name: 'sdk-net--ubuntu-20'
    displayName: Linux
    strategy:
      maxParallel: $[variables.maxParallel]
      matrix:
        JDK 11: # always run Java 11
          JAVA_VERSION: 11
        JDK 8: # only run Java 8 in nightly or CI builds.
          JAVA_VERSION: 8
    steps:
      - task: CmdLine@2
        displayName: 'Print Linux version'
        inputs:
          script: 'cat /etc/*release'

      - powershell: ./vsts/echo_versions.ps1
        displayName: 'Echo Versions'
        env:
          JAVA_VERSION: $(JAVA_VERSION)
          COMMIT_FROM: $(COMMIT_FROM)

      - task : Docker@2
        displayName: "Login to ACR"
        inputs:
          command: "login"
          containerRegistry: "Azure IoT ACR"

      - task: PowerShell@2
        displayName: 'Cert Validation - Setup Certificate Proxy'
        inputs:
          targetType: 'inline'
          script: |
            Write-Host "============"
            Write-Host "check docker"
            Write-Host "============"
            docker images
            docker ps -a
            gci -Path $(Build.SourcesDirectory)/iot-e2e-tests/docker -Recurse -Force -Name
            Write-Host "==========================="
            Write-Host "uncompress self-signed cert"
            Write-Host "==========================="
            gzip -d -N -S ".bin" $(Build.SourcesDirectory)/iot-e2e-tests/docker/haproxy/haproxy.bin
            gci -Path $(Build.SourcesDirectory)/iot-e2e-tests/docker -Recurse -Force -Name
            Write-Host "====================="
            Write-Host "Docker run containers"
            Write-Host "====================="
            docker run -h invalidcertgde1.westus.cloudapp.azure.com --name invalid-gde --expose=443 --expose=5671 --expose=8883 -v $(Build.SourcesDirectory)/iot-e2e-tests/docker/haproxy:/usr/local/etc/haproxy:ro -d aziotacr.azurecr.io/haproxy haproxy -f /usr/local/etc/haproxy/haproxygde.cfg
            docker run -h invalidcertdps1.westus.cloudapp.azure.com --name invalid-dps --expose=443 --expose=5671 --expose=8883 -v $(Build.SourcesDirectory)/iot-e2e-tests/docker/haproxy:/usr/local/etc/haproxy:ro -d aziotacr.azurecr.io/haproxy haproxy -f /usr/local/etc/haproxy/haproxydps.cfg
            docker run -h invalidcertiothub1.westus.cloudapp.azure.com --name invalid-hub --expose=443 --expose=5671 --expose=8883 -v $(Build.SourcesDirectory)/iot-e2e-tests/docker/haproxy:/usr/local/etc/haproxy:ro -d aziotacr.azurecr.io/haproxy haproxy -f /usr/local/etc/haproxy/haproxyhub.cfg
            docker ps -a

      - task: Bash@3
        displayName: 'Cert Validation - Setup local hostname'
        name: CVTEST_NET
        inputs:
          targetType: 'inline'
          script: |
            echo "==============="
            echo "Inspect network"
            echo "==============="
            ip -4 addr
            export CVTEST_HOST_IP=$(ip -4 addr show eth0 | grep -Po 'inet \K[\d.]+')
            export CVTEST_HOST_NETWORK=$(ip -4 addr show eth0 | grep -Po 'inet \K[\d.]+/*\d.')
            export CVTEST_HOST_SUBNET=$(ip -4 addr show eth0 | grep -Po 'inet \K[\d.]+' | grep -Po '[\d]{1,3}.[\d]{1,3}.[\d]{1,3}')
            export CVTEST_HOST_SUBNET_MASK=$(ip -4 addr show eth0 | grep -Po 'inet \K[\d.]+/*\d.' | grep -Po '/[\d]{1,2}')
            export CVTEST_CONTAINER_IP=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
            echo "HOST=$CVTEST_HOST_IP"
            echo "HOST NETWORK=$CVTEST_HOST_NETWORK"
            echo "HOST SUBNET=$CVTEST_HOST_SUBNET"
            echo "HOST SUBNET MASK=$CVTEST_HOST_SUBNET_MASK"
            echo "CONTAINER=$CVTEST_CONTAINER_IP"
            #echo "##vso[task.setvariable variable=AGENT_HOST;isoutput=true;]$CVTEST_HOST_IP"
            #echo "##vso[task.setvariable variable=AGENT_NETWORK;isoutput=true;]$CVTEST_HOST_NETWORK"
            #echo "##vso[task.setvariable variable=AGENT_SUBNET;isoutput=true;]$CVTEST_HOST_SUBNET"
            #echo "##vso[task.setvariable variable=AGENT_SUBNET_MASK;isoutput=true;]$CVTEST_HOST_SUBNET_MASK"
            #echo "##vso[task.setvariable variable=AGENT_CONTAINER;isoutput=true;]$CVTEST_CONTAINER_IP"
            export CVTEST_GDE_IP=$(docker inspect invalid-gde | grep -Po -m 1 '"IPAddress": "\K[\d.]+')
            export CVTEST_DPS_IP=$(docker inspect invalid-dps | grep -Po -m 1 '"IPAddress": "\K[\d.]+')
            export CVTEST_HUB_IP=$(docker inspect invalid-hub | grep -Po -m 1 '"IPAddress": "\K[\d.]+')
            echo "invalid-gde=$CVTEST_GDE_IP"
            echo "invalid-dps=$CVTEST_DPS_IP"
            echo "invalid-hub=$CVTEST_HUB_IP"
            #echo "##vso[task.setvariable variable=AGENT_GDE_IP;isoutput=true;]$CVTEST_GDE_IP"
            #echo "##vso[task.setvariable variable=AGENT_DPS_IP;isoutput=true;]$CVTEST_DPS_IP"
            #echo "##vso[task.setvariable variable=AGENT_HUB_IP;isoutput=true;]$CVTEST_HUB_IP"
            echo "================="
            echo "Update hosts file"
            echo "================="
            sudo bash -c 'mv /etc/hosts /etc/hosts.org'
            sudo bash -c 'cp /etc/hosts.org /etc/hosts'
            echo "$CVTEST_GDE_IP invalidcertgde1.westus.cloudapp.azure.com"
            echo "$CVTEST_DPS_IP invalidcertdps1.westus.cloudapp.azure.com"
            echo "$CVTEST_HUB_IP invalidcertiothub1.westus.cloudapp.azure.com"
            echo "" >> /tmp/hosts.cvtest
            echo "# Local host for invalid cert test" >> /tmp/hosts.cvtest
            echo "$CVTEST_GDE_IP invalidcertgde1.westus.cloudapp.azure.com" >> /tmp/hosts.cvtest
            echo "$CVTEST_DPS_IP invalidcertdps1.westus.cloudapp.azure.com" >> /tmp/hosts.cvtest
            echo "$CVTEST_HUB_IP invalidcertiothub1.westus.cloudapp.azure.com" >> /tmp/hosts.cvtest
            sudo bash -c 'cat /tmp/hosts.cvtest >> /etc/hosts'
            cat /etc/hosts
            echo "====================="
            echo "Ping containers (URL)"
            echo "====================="
            docker ps -a
            route
            ping -c 2 invalidcertgde1.westus.cloudapp.azure.com
            ping -c 2 invalidcertdps1.westus.cloudapp.azure.com
            ping -c 2 invalidcertiothub1.westus.cloudapp.azure.com

      - task: Docker@1
        displayName: 'Start TPM Simulator'
        inputs:
          containerregistrytype: 'Container Registry'
          command: 'Run an image'
          imageName: aziotbld/testtpm
          containerName: 'testtpm-instance'
          ports: |
            127.0.0.1:2321:2321
            127.0.0.1:2322:2322
          restartPolicy: unlessStopped

      - powershell: ./vsts/build_repo.ps1
        displayName: 'Build and Test'
        env:
          JAVA_VERSION: $(JAVA_VERSION)
          IOT_DPS_CONNECTION_STRING: $(IOT-DPS-CONNECTION-STRING)
          IOT_DPS_ID_SCOPE: $(IOT-DPS-ID-SCOPE)
          IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
          DPS_GLOBALDEVICEENDPOINT_INVALIDCERT: $(DPS-GLOBALDEVICEENDPOINT-INVALIDCERT)
          PROVISIONING_CONNECTION_STRING_INVALIDCERT: $(PROVISIONING-CONNECTION-STRING-INVALIDCERT)
          IS_BASIC_TIER_HUB: $(IS-BASIC-TIER-HUB)
          TARGET_BRANCH: $(System.PullRequest.TargetBranch)
          RECYCLE_TEST_IDENTITIES: true

      - task: CopyFiles@2
        condition: always()
        displayName: 'Copy Test Results to Artifact Staging Directory'
        inputs:
          SourceFolder: '$(Build.SourcesDirectory)'
          Contents: |
            **/*.trx
            **/*.xml
          TargetFolder: '$(Build.ArtifactStagingDirectory)'
        continueOnError: true

      - task: PublishBuildArtifacts@1
        displayName: 'Publish Artifact Staging Directory'
        condition: always()
        continueOnError: true

      - task: PublishTestResults@2
        displayName: 'Publish Test Results'
        condition: always()
        inputs:
          mergeTestResults: true
          testRunTitle: "Linux JDK $(JAVA_VERSION) (Attempt $(System.JobAttempt))"
        continueOnError: true

      - task: ComponentGovernanceComponentDetection@0
        displayName: Component Governance Detection
        condition: always()
        inputs:
          scanType: 'Register'
          verbosity: 'Verbose'
          alertWarningLevel: 'Low' # The task will present a warning, but will not cause the build to fail

  ### Android, Multi configuration build (Multiple different test groups to cover) ###
  - job: AndroidBuild
    timeoutInMinutes: 30
    pool:
      vmImage: ubuntu-latest
    displayName: Android Build

    steps:
      - powershell: ./vsts/echo_versions.ps1
        displayName: 'Echo Versions'
        env:
          COMMIT_FROM: $(COMMIT_FROM)
          JAVA_VERSION: 8 # Android build doesn't need to run for both JDK 8 and 11
        condition: always()

      - powershell: ./vsts/build_e2e_tests.ps1
        displayName: 'E2E Tests Build'
        condition: always()

      - powershell: ./vsts/gradle_build.ps1
        displayName: 'Gradle Build'
        env:
          IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
          DEVICE_PROVISIONING_SERVICE_ID_SCOPE: $(IOT-DPS-ID-SCOPE)
          IOT_DPS_CONNECTION_STRING: $(IOT-DPS-CONNECTION-STRING)
          INVALID_DEVICE_PROVISIONING_SERVICE_GLOBAL_ENDPOINT: $(DPS-GLOBALDEVICEENDPOINT-INVALIDCERT)
          IS_BASIC_TIER_HUB: $(IS-BASIC-TIER-HUB)
          TARGET_BRANCH: $(System.PullRequest.TargetBranch)
          RECYCLE_TEST_IDENTITIES: true
        condition: always()

      - task: CopyFiles@2
        displayName: 'Copy Test Results to Artifact Staging Directory'
        inputs:
          SourceFolder: '$(Build.SourcesDirectory)/iot-e2e-tests/android/app/build/outputs/apk'
          Contents: |
            *.*
          TargetFolder: '$(Build.ArtifactStagingDirectory)'
        continueOnError: true
        condition: always()

      - task: PublishPipelineArtifact@0
        displayName: 'Publish APKs to test in next job'
        inputs:
          artifactName: 'androidBuildFiles'
          targetPath: 'iot-e2e-tests/android/app/build/outputs/apk'

      - task: ComponentGovernanceComponentDetection@0
        displayName: Component Governance Detection
        inputs:
          scanType: 'Register'
          verbosity: 'Verbose'
          alertWarningLevel: 'Low' # The task will present a warning, but will not cause the build to fail
        condition: always()

  - job: AndroidTest
    timeoutInMinutes: 50
    pool:
      vmImage: 'macOS-latest'
    strategy:
      maxParallel: 12
      matrix:
        TestGroup1:
          ANDROID_TEST_GROUP_ID: TestGroup1
        TestGroup2:
          ANDROID_TEST_GROUP_ID: TestGroup2
        TestGroup3:
          ANDROID_TEST_GROUP_ID: TestGroup3
        TestGroup4:
          ANDROID_TEST_GROUP_ID: TestGroup4
        TestGroup5:
          ANDROID_TEST_GROUP_ID: TestGroup5
        TestGroup6:
          ANDROID_TEST_GROUP_ID: TestGroup6
        TestGroup7:
          ANDROID_TEST_GROUP_ID: TestGroup7
        TestGroup8:
          ANDROID_TEST_GROUP_ID: TestGroup8
        TestGroup9:
          ANDROID_TEST_GROUP_ID: TestGroup9
        TestGroup10:
          ANDROID_TEST_GROUP_ID: TestGroup10
        TestGroup11:
          ANDROID_TEST_GROUP_ID: TestGroup11
        TestGroup12:
          ANDROID_TEST_GROUP_ID: TestGroup12

    displayName: Android Test
    dependsOn: AndroidBuild
    steps:
      - task: PowerShell@2
        displayName: 'determine if testing needed'
        condition: always()
        inputs:
          targetType: 'filePath'
          filePath: ./vsts/determine_if_android_test_group_needs_to_run.ps1
        env:
          TEST_GROUP_ID: $(ANDROID_TEST_GROUP_ID)
          IS_BASIC_TIER_HUB: $(IS-BASIC-TIER-HUB)
          TARGET_BRANCH: $(System.PullRequest.TargetBranch)

      - task: DownloadPipelineArtifact@0
        displayName: "Download APKs to test from previous job"
        condition: eq(variables['task.android.needToRunTestGroup'], 'yes')
        inputs:
          artifactName: 'androidBuildFiles'
          targetPath: $(Build.SourcesDirectory)/iot-e2e-tests/android/app/build/outputs/apk

      - task: Bash@3
        condition: eq(variables['task.android.needToRunTestGroup'], 'yes')
        displayName: 'Start Android Emulator'
        timeoutInMinutes: 15
        continueOnError: false
        inputs:
          targetType: 'filePath'
          filePath: '$(Build.SourcesDirectory)/vsts/StartEmulator.sh'

      - task: Bash@3
        #only run tests on emulator if tests should be run, and if the emulator boot up was successful
        condition: and(succeeded(), eq(variables['task.android.needToRunTestGroup'], 'yes'))
        displayName: 'Run tests on emulator'
        timeoutInMinutes: 45
        inputs:
          targetType: 'filePath'
          filePath: '$(Build.SourcesDirectory)/vsts/RunTestsOnEmulator.sh'
        env:
          TEST_GROUP_ID: $(ANDROID_TEST_GROUP_ID)

      - task : ComponentGovernanceComponentDetection@0
        displayName : Component Governance Detection
        inputs :
          scanType : 'Register'
          verbosity : 'Verbose'
          alertWarningLevel : 'Low' # The task will present a warning, but will not cause the build to fail
        condition : always()