eng/common/scripts/job-matrix/job-matrix-functions.ps1 (672 lines of code) (raw):

Set-StrictMode -Version "4.0" class MatrixConfig { [PSCustomObject]$displayNames [Hashtable]$displayNamesLookup [PSCustomObject]$matrix [MatrixParameter[]]$matrixParameters [Array]$include [Array]$exclude } class MatrixParameter { MatrixParameter([String]$name, [System.Object]$value) { $this.Value = $value $this.Name = $name } [System.Object]$Value [System.Object]$Name Set($value, [String]$keyRegex = '') { if ($this.Value -is [PSCustomObject]) { $set = $false foreach ($prop in $this.Value.PSObject.Properties) { if ($prop.Name -match $keyRegex) { $prop.Value = $value $set = $true break } } if (!$set) { throw "Property `"$keyRegex`" does not exist for MatrixParameter." } } else { $this.Value = $value } } [System.Object]Flatten() { if ($this.Value -is [PSCustomObject]) { return $this.Value.PSObject.Properties | ForEach-Object { [MatrixParameter]::new($_.Name, $_.Value) } } elseif ($this.Value -is [Array]) { return $this.Value | ForEach-Object { [MatrixParameter]::new($this.Name, $_) } } else { return $this } } [Int]Length() { if ($this.Value -is [PSCustomObject]) { return ($this.Value.PSObject.Properties | Measure-Object).Count } elseif ($this.Value -is [Array]) { return $this.Value.Length } else { return 1 } } [String]CreateDisplayName([Hashtable]$displayNamesLookup) { if ($null -eq $this.Value) { $displayName = "" } elseif ($this.Value -is [PSCustomObject]) { $displayName = $this.Name } else { $displayName = $this.Value.ToString() } if ($displayNamesLookup -and $displayNamesLookup.ContainsKey($displayName)) { $displayName = $displayNamesLookup[$displayName] } # Matrix naming restrictions: # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration $displayName = $displayName -replace "[^A-Za-z0-9_]", "" return $displayName } } . (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1) $IMPORT_KEYWORD = '$IMPORT' function GenerateMatrix( [MatrixConfig]$config, [String]$selectFromMatrixType, [String]$displayNameFilter = ".*", [Array]$filters = @(), [Array]$replace = @(), [Array]$nonSparseParameters = @(), [Switch]$skipEnvironmentVariables ) { $result = ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup $matrixParameters = $result.Matrix $importedMatrix = $result.ImportedMatrix $combinedDisplayNameLookup = $result.DisplayNamesLookup if ($selectFromMatrixType -eq "sparse") { $matrix = GenerateSparseMatrix $matrixParameters $config.displayNamesLookup $nonSparseParameters } elseif ($selectFromMatrixType -eq "all") { $matrix = GenerateFullMatrix $matrixParameters $config.displayNamesLookup } else { throw "Matrix generator not implemented for selectFromMatrixType: '$selectFromMatrixType'" } # Combine with imported after matrix generation, since a sparse selection should result in a full combination of the # top level and imported sparse matrices (as opposed to a sparse selection of both matrices). if ($importedMatrix) { $matrix = CombineMatrices $matrix $importedMatrix $combinedDisplayNameLookup } if ($config.exclude) { $matrix = ProcessExcludes $matrix $config.exclude } if ($config.include) { $matrix = ProcessIncludes $config $matrix $selectFromMatrixType } $matrix = FilterMatrix $matrix $filters $matrix = ProcessReplace $matrix $replace $combinedDisplayNameLookup if (!$skipEnvironmentVariables) { $matrix = ProcessEnvironmentVariableReferences $matrix $combinedDisplayNameLookup } $matrix = FilterMatrixDisplayName $matrix $displayNameFilter return $matrix } function ProcessNonSparseParameters( [MatrixParameter[]]$parameters, [Array]$nonSparseParameters ) { if (!$nonSparseParameters) { return $parameters, $null } $sparse = [MatrixParameter[]]@() $nonSparse = [MatrixParameter[]]@() foreach ($param in $parameters) { if ($null -eq $param){ continue } if ($param.Name -in $nonSparseParameters) { $nonSparse += $param } else { $sparse += $param } } return $sparse, $nonSparse } function FilterMatrixDisplayName([array]$matrix, [string]$filter) { return $matrix | Where-Object { $_ } | ForEach-Object { if ($_.ContainsKey("Name") -and $_.Name -match $filter) { return $_ } } } # Filters take the format of key=valueregex,key2=valueregex2 function FilterMatrix([array]$matrix, [array]$filters) { $matrix = $matrix | ForEach-Object { if (MatchesFilters $_ $filters) { return $_ } } return $matrix } function MatchesFilters([hashtable]$entry, [array]$filters) { foreach ($filter in $filters) { $key, $regex = ParseFilter $filter # Default all regex checks to go against empty string when keys are missing. # This simplifies the filter syntax/interface to be regex only. $value = "" if ($null -ne $entry -and $entry.ContainsKey("parameters") -and $entry.parameters.Contains($key)) { $value = $entry.parameters[$key] } if ($value -notmatch $regex) { return $false } } return $true } function ParseFilter([string]$filter) { # Lazy match key in case value contains '=' if ($filter -match "(.+?)=(.+)") { $key = $matches[1] $regex = $matches[2] return $key, $regex } else { throw "Invalid filter: `"${filter}`", expected <key>=<regex> format" } } function GetMatrixConfigFromFile([String] $config) { [MatrixConfig]$config = try { GetMatrixConfigFromJson $config } catch { GetMatrixConfigFromYaml $config } return $config } function GetMatrixConfigFromYaml([String] $yamlConfig) { Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module # ConvertTo then from json is to make sure the nested values are in PSCustomObject [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json return GetMatrixConfig $config } function GetMatrixConfigFromJson([String]$jsonConfig) { [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json return GetMatrixConfig $config } # Importing the JSON as PSCustomObject preserves key ordering, # whereas ConvertFrom-Json -AsHashtable does not function GetMatrixConfig([MatrixConfig]$config) { $config.matrixParameters = @() $config.displayNamesLookup = @{} $include = [MatrixParameter[]]@() $exclude = [MatrixParameter[]]@() if ($null -ne $config.displayNames) { $config.displayNames.PSObject.Properties | ForEach-Object { $config.displayNamesLookup.Add($_.Name, $_.Value) } } if ($null -ne $config.matrix) { $config.matrixParameters = PsObjectToMatrixParameterArray $config.matrix } foreach ($includeMatrix in $config.include) { $include += , @(PsObjectToMatrixParameterArray $includeMatrix) } foreach ($excludeMatrix in $config.exclude) { $exclude += , @(PsObjectToMatrixParameterArray $excludeMatrix) } $config.include = $include $config.exclude = $exclude return $config } function PsObjectToMatrixParameterArray([PSCustomObject]$obj) { if ($obj -eq $null) { return $null } return $obj.PSObject.Properties | ForEach-Object { [MatrixParameter]::new($_.Name, $_.Value) } } function ProcessExcludes([Array]$matrix, [Array]$excludes) { $deleteKey = "%DELETE%" $exclusionMatrix = @() foreach ($exclusion in $excludes) { $full = GenerateFullMatrix $exclusion $exclusionMatrix += $full } foreach ($element in $matrix) { foreach ($exclusion in $exclusionMatrix) { $match = MatrixElementMatch $element.parameters $exclusion.parameters if ($match) { $element.parameters[$deleteKey] = $true } } } return $matrix | Where-Object { !$_.parameters.Contains($deleteKey) } } function ProcessIncludes([MatrixConfig]$config, [Array]$matrix) { $inclusionMatrix = @() foreach ($inclusion in $config.include) { $full = GenerateFullMatrix $inclusion $config.displayNamesLookup $inclusionMatrix += $full } return $matrix + $inclusionMatrix } function ParseReplacement([String]$replacement) { $parsed = '', '', '' $idx = 0 $escaped = $false $operators = '=', '/' $err = "Invalid replacement syntax, expecting <key>=<value>/<replace>" foreach ($c in $replacement -split '') { if ($idx -ge $parsed.Length) { throw $err } if (!$escaped -and $c -in $operators) { $idx++ } else { $parsed[$idx] += $c } $escaped = $c -eq '\' } if ($idx -lt $parsed.Length - 1) { throw $err } $replace = $parsed[2] -replace "\\([$($operators -join '')])", '$1' return @{ "key" = '^' + $parsed[0] + '$' # Force full matches only. "value" = '^' + $parsed[1] + '$' "replace" = $replace } } function ProcessReplace { param( [Array]$matrix, [Array]$replacements, [Hashtable]$displayNamesLookup ) if (!$replacements) { return $matrix } $replaceMatrix = @() foreach ($element in $matrix) { if (!$element -or $element.Count -eq 0) { continue } $replacement = [MatrixParameter[]]@() if (!$element -or $element.Count -eq 0) { continue } foreach ($perm in $element._permutation) { $replace = $perm # Iterate nested permutations or run once for singular values (int, string, bool) foreach ($flattened in $perm.Flatten()) { foreach ($query in $replacements) { $parsed = ParseReplacement $query if ($flattened.Name -match $parsed.key -and $flattened.Value -match $parsed.value) { # In most cases, this will just swap one value for another, however -replace # is used here in order to support replace values which may use regex capture groups # e.g. 'foo-1' -replace '(foo)-1', '$1-replaced' $replaceValue = $flattened.Value -replace $parsed.value, $parsed.replace $perm.Set($replaceValue, $parsed.key) break } } } $replacement += $perm } $replaceMatrix += CreateMatrixCombinationScalar $replacement $displayNamesLookup } return $replaceMatrix } function ProcessEnvironmentVariableReferences([array]$matrix, $displayNamesLookup) { $updatedMatrix = @() $missingEnvVars = @{} foreach ($element in $matrix) { $updated = [MatrixParameter[]]@() if (!$element -or $element.Count -eq 0) { continue } foreach ($perm in $element._permutation) { # Iterate nested permutations or run once for singular values (int, string, bool) foreach ($flattened in $perm.Flatten()) { if ($flattened.Value -is [string] -and $flattened.Value.StartsWith("env:")) { $envKey = $flattened.Value.Replace("env:", "") $value = [System.Environment]::GetEnvironmentVariable($envKey) if (!$value) { $missingEnvVars[$envKey] = $true } $perm.Set($value, $flattened.Name) } } $updated += $perm } $updatedMatrix += CreateMatrixCombinationScalar $updated $displayNamesLookup } if ($missingEnvVars.Count -gt 0) { throw "Environment variables '$($missingEnvVars.Keys -join ", ")' were empty or not found." } return $updatedMatrix } function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$nonSparseParameters, [Hashtable]$displayNamesLookup) { $importPath = "" $matrix = $matrix | ForEach-Object { if ($_.Name -ne $IMPORT_KEYWORD) { return $_ } else { $importPath = $_.Value } } if ((!$matrix -and !$importPath) -or !$importPath) { return [PSCustomObject]@{ Matrix = $matrix ImportedMatrix = @() DisplayNamesLookup = $displayNamesLookup } } if (!(Test-Path $importPath)) { Write-Error "`$IMPORT path '$importPath' does not exist. Current dir: $(Get-Location)" exit 1 } $importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath) # Add skipEnvironmentVariables so we don't process environment variables on import # because we want top level filters to work against the the env key, not the value. # The environment variables will get resolved after the import. $importedMatrix = GenerateMatrix ` -config $importedMatrixConfig ` -selectFromMatrixType $selection ` -nonSparseParameters $nonSparseParameters ` -skipEnvironmentVariables $combinedDisplayNameLookup = $importedMatrixConfig.displayNamesLookup foreach ($lookup in $displayNamesLookup.GetEnumerator()) { $combinedDisplayNameLookup[$lookup.Name] = $lookup.Value } return [PSCustomObject]@{ Matrix = $matrix ?? @() ImportedMatrix = $importedMatrix DisplayNamesLookup = $combinedDisplayNameLookup } } function CombineMatrices([Array]$matrix1, [Array]$matrix2, [Hashtable]$displayNamesLookup = @{}) { $combined = @() if (!$matrix1) { return $matrix2 } if (!$matrix2) { return $matrix1 } foreach ($entry1 in $matrix1) { foreach ($entry2 in $matrix2) { $combined += CreateMatrixCombinationScalar ($entry1._permutation + $entry2._permutation) $displayNamesLookup } } return $combined } function MatrixElementMatch([System.Collections.Specialized.OrderedDictionary]$source, [System.Collections.Specialized.OrderedDictionary]$target) { if ($target.Count -eq 0) { return $false } foreach ($key in $target.Keys) { if (!$source.Contains($key) -or $source[$key] -ne $target[$key]) { return $false } } return $true } function CloneOrderedDictionary([System.Collections.Specialized.OrderedDictionary]$dictionary) { $newDictionary = [Ordered]@{} foreach ($element in $dictionary.GetEnumerator()) { $newDictionary[$element.Name] = $element.Value } return $newDictionary } function SerializePipelineMatrix([Array]$matrix) { $pipelineMatrix = [Ordered]@{} foreach ($entry in $matrix) { if ($pipelineMatrix.Contains($entry.name)) { Write-Warning "Found duplicate configurations for job `"$($entry.name)`". Multiple values may have been replaced with the same value." continue } $pipelineMatrix.Add($entry.name, [Ordered]@{}) foreach ($key in $entry.parameters.Keys) { $pipelineMatrix[$entry.name].Add($key, $entry.parameters[$key]) } } return @{ compressed = $pipelineMatrix | ConvertTo-Json -Compress ; pretty = $pipelineMatrix | ConvertTo-Json; } } function GenerateSparseMatrix( [MatrixParameter[]]$parameters, [Hashtable]$displayNamesLookup, [Array]$nonSparseParameters = @() ) { $parameters, $nonSparse = ProcessNonSparseParameters $parameters $nonSparseParameters $dimensions = GetMatrixDimensions $parameters $matrix = GenerateFullMatrix $parameters $displayNamesLookup $sparseMatrix = @() [array]$indexes = GetSparseMatrixIndexes $dimensions foreach ($idx in $indexes) { $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions } if ($nonSparse) { $allOfMatrix = GenerateFullMatrix $nonSparse $displayNamesLookup return CombineMatrices $allOfMatrix $sparseMatrix $displayNamesLookup } return $sparseMatrix } function GetSparseMatrixIndexes([Array]$dimensions) { $size = ($dimensions | Measure-Object -Maximum).Maximum $indexes = @() # With full matrix, retrieve items by doing diagonal lookups across the matrix N times. # For example, given a matrix with dimensions 3, 2, 2: # 0, 0, 0 # 1, 1, 1 # 2, 2, 2 # 3, 0, 0 <- 3, 3, 3 wraps to 3, 0, 0 given the dimensions for ($i = 0; $i -lt $size; $i++) { $idx = @() for ($j = 0; $j -lt $dimensions.Length; $j++) { $idx += $i % $dimensions[$j] } $indexes += , $idx } return , $indexes } function GenerateFullMatrix( [MatrixParameter[]] $parameters, [Hashtable]$displayNamesLookup = @{} ) { # Handle when the config does not have a matrix specified (e.g. only the include field is specified) if (!$parameters) { return @() } $matrix = [System.Collections.ArrayList]::new() InitializeMatrix $parameters $displayNamesLookup $matrix return $matrix } function CreateMatrixCombinationScalar([MatrixParameter[]]$permutation, [Hashtable]$displayNamesLookup = @{}) { $names = @() $flattenedParameters = [Ordered]@{} foreach ($entry in $permutation) { $nameSegment = "" # Unwind nested permutations or run once for singular values (int, string, bool) foreach ($param in $entry.Flatten()) { if ($flattenedParameters.Contains($param.Name)) { throw "Found duplicate parameter `"$($param.Name)`" when creating matrix combination." } $flattenedParameters.Add($param.Name, $param.Value) } $nameSegment = $entry.CreateDisplayName($displayNamesLookup) if ($nameSegment) { $names += $nameSegment } } # The maximum allowed matrix name length is 100 characters $name = $names -join "_" if ($name -and $name[0] -match "^[0-9]") { $name = "job_" + $name # Azure Pipelines only supports job names starting with letters } if ($name.Length -gt 100) { $name = $name[0..99] -join "" } return @{ name = $name parameters = $flattenedParameters # Keep the original permutation around in case we need to re-process this entry when transforming the matrix _permutation = $permutation } } function InitializeMatrix { param( [MatrixParameter[]]$parameters, [Hashtable]$displayNamesLookup, [System.Collections.ArrayList]$permutations, $permutation = [MatrixParameter[]]@() ) $head, $tail = $parameters if (!$head) { $entry = CreateMatrixCombinationScalar $permutation $displayNamesLookup $permutations.Add($entry) | Out-Null return } # This behavior implicitly treats non-array values as single elements foreach ($param in $head.Flatten()) { $newPermutation = $permutation + $param InitializeMatrix $tail $displayNamesLookup $permutations $newPermutation } } function GetMatrixDimensions([MatrixParameter[]]$parameters) { $dimensions = @() foreach ($param in $parameters) { if ($null -eq $param){ continue } $dimensions += $param.Length() } return $dimensions } function SetNdMatrixElement { param( $element, [ValidateNotNullOrEmpty()] [Array]$idx, [ValidateNotNullOrEmpty()] [Array]$matrix, [ValidateNotNullOrEmpty()] [Array]$dimensions ) if ($idx.Length -ne $dimensions.Length) { throw "Matrix index query $($idx.Length) must be the same length as its dimensions $($dimensions.Length)" } $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions $matrix[$arrayIndex] = $element } function GetNdMatrixArrayIndex { param( [ValidateNotNullOrEmpty()] [Array]$idx, [ValidateNotNullOrEmpty()] [Array]$dimensions ) if ($idx.Length -ne $dimensions.Length) { throw "Matrix index query length ($($idx.Length)) must be the same as dimension length ($($dimensions.Length))" } $stride = 1 # Commented out does lookup with wrap handling # $index = $idx[$idx.Length-1] % $dimensions[$idx.Length-1] $index = $idx[$idx.Length - 1] for ($i = $dimensions.Length - 1; $i -ge 1; $i--) { $stride *= $dimensions[$i] # Commented out does lookup with wrap handling # $index += ($idx[$i-1] % $dimensions[$i-1]) * $stride $index += $idx[$i - 1] * $stride } return $index } function GetNdMatrixElement { param( [ValidateNotNullOrEmpty()] [Array]$idx, [ValidateNotNullOrEmpty()] [Array]$matrix, [ValidateNotNullOrEmpty()] [Array]$dimensions ) $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions return $matrix[$arrayIndex] } function GetNdMatrixIndex { param( [int]$index, [ValidateNotNullOrEmpty()] [Array]$dimensions ) $matrixIndex = @() $stride = 1 for ($i = $dimensions.Length - 1; $i -ge 1; $i--) { $stride *= $dimensions[$i] $page = [math]::floor($index / $stride) % $dimensions[$i - 1] $matrixIndex = , $page + $matrixIndex } $col = $index % $dimensions[$dimensions.Length - 1] $matrixIndex += $col return $matrixIndex } # # # # # # # # # # # # # # # # # # # # # # # # # # # # # The below functions are non-dynamic examples that # # help explain the above N-dimensional algorithm # # # # # # # # # # # # # # # # # # # # # # # # # # # # # function Get4dMatrixElement([Array]$idx, [Array]$matrix, [Array]$dimensions) { $stride1 = $idx[0] * $dimensions[1] * $dimensions[2] * $dimensions[3] $stride2 = $idx[1] * $dimensions[2] * $dimensions[3] $stride3 = $idx[2] * $dimensions[3] $stride4 = $idx[3] return $matrix[$stride1 + $stride2 + $stride3 + $stride4] } function Get4dMatrixIndex([int]$index, [Array]$dimensions) { $stride1 = $dimensions[3] $stride2 = $dimensions[2] $stride3 = $dimensions[1] $page1 = [math]::floor($index / $stride1) % $dimensions[2] $page2 = [math]::floor($index / ($stride1 * $stride2)) % $dimensions[1] $page3 = [math]::floor($index / ($stride1 * $stride2 * $stride3)) % $dimensions[0] $remainder = $index % $dimensions[3] return @($page3, $page2, $page1, $remainder) } function GenerateMatrixForConfig { param ( [Parameter(Mandatory = $true)][string] $ConfigPath, [Parameter(Mandatory = $true)][string] $Selection, [Parameter(Mandatory = $false)][string] $DisplayNameFilter, [Parameter(Mandatory = $false)][array] $Filters, [Parameter(Mandatory = $false)][array] $Replace, [Parameter(Mandatory = $false)][Array] $NonSparseParameters = @() ) $matrixFile = Join-Path $PSScriptRoot ".." ".." ".." ".." $ConfigPath $resolvedMatrixFile = Resolve-Path $matrixFile $config = GetMatrixConfigFromFile (Get-Content $resolvedMatrixFile -Raw) # Strip empty string filters in order to be able to use azure pipelines yaml join() $Filters = $Filters | Where-Object { $_ } [array]$matrix = GenerateMatrix ` -config $config ` -selectFromMatrixType $Selection ` -displayNameFilter $DisplayNameFilter ` -filters $Filters ` -replace $Replace ` -nonSparseParameters $NonSparseParameters return , $matrix }