From ca1692e375fdba9ebb56c21416ec4c28b0e68fe4 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Tue, 16 Mar 2021 16:46:41 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 1463 (#1919) * Add 'replace' support. Improve handling of dynamic parameter types. * Support display name import and overrides * Support regex capture groups for replace. Force fully matching regex. Co-authored-by: Ben Broderick Phillips --- .../jobs/archetype-sdk-tests-generate.yml | 4 + .../scripts/job-matrix/Create-JobMatrix.ps1 | 2 + eng/common/scripts/job-matrix/README.md | 72 +++- .../job-matrix/job-matrix-functions.ps1 | 353 ++++++++++++------ .../job-matrix/samples/matrix-test.yml | 7 +- .../scripts/job-matrix/samples/matrix.json | 8 +- ...ob-matrix-functions.modification.tests.ps1 | 215 +++++++++-- .../tests/job-matrix-functions.tests.ps1 | 41 +- .../job-matrix/tests/test-import-matrix.json | 3 + 9 files changed, 533 insertions(+), 172 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml b/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml index 9f7ca629c..893bc87a7 100644 --- a/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml +++ b/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml @@ -10,6 +10,9 @@ parameters: - name: MatrixFilters type: object default: [] +- name: MatrixReplace + type: object + default: {} - name: JobTemplatePath type: string # Set this to false to do a full checkout for private repositories with the azure pipelines service connection @@ -57,6 +60,7 @@ jobs: -Selection ${{ config.Selection }} -DisplayNameFilter "$(displayNameFilter)" -Filters "${{ join('","', parameters.MatrixFilters) }}","container=^$","SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}" + -Replace "${{ join('","', parameters.MatrixReplace) }}" -NonSparseParameters "${{ join('","', config.NonSparseParameters) }}" displayName: Generate VM Job Matrix ${{ config.Name }} name: generate_vm_job_matrix_${{ config.Name }} diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 index 3e075c3e0..729342ca6 100644 --- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 @@ -13,6 +13,7 @@ param ( [Parameter(Mandatory=$True)][string] $Selection, [Parameter(Mandatory=$False)][string] $DisplayNameFilter, [Parameter(Mandatory=$False)][array] $Filters, + [Parameter(Mandatory=$False)][array] $Replace, [Parameter(Mandatory=$False)][array] $NonSparseParameters ) @@ -27,6 +28,7 @@ $Filters = $Filters | Where-Object { $_ } -selectFromMatrixType $Selection ` -displayNameFilter $DisplayNameFilter ` -filters $Filters ` + -replace $Replace ` -nonSparseParameters $NonSparseParameters $serialized = SerializePipelineMatrix $matrix diff --git a/eng/common/scripts/job-matrix/README.md b/eng/common/scripts/job-matrix/README.md index 4f468e4cc..13bf6cf43 100644 --- a/eng/common/scripts/job-matrix/README.md +++ b/eng/common/scripts/job-matrix/README.md @@ -14,6 +14,7 @@ * [include/exclude](#includeexclude) * [displayNames](#displaynames-1) * [Filters](#filters) + * [Replace](#replace-values) * [NonSparseParameters](#nonsparseparameters) * [Under the hood](#under-the-hood) * [Testing](#testing) @@ -54,6 +55,7 @@ jobs: Location: eastus2 Cloud: Public MatrixFilters: [] + MatrixReplace: [] ``` ## Matrix config file syntax @@ -166,7 +168,8 @@ To import a matrix, add a parameter with the key `$IMPORT`: ``` Importing can be useful, for example, in cases where there is a shared base matrix, but there is a need to run it -once for each instance of a language version. +once for each instance of a language version. Importing does not support overriding duplicate parameters. To achieve +this, use the [Replace](#replace-values) argument instead. The processing order is as follows: @@ -376,7 +379,7 @@ The logic for generating display names works like this: #### Filters -Filters can be passed to the matrix as an array of strings, each matching the format of =. When a matrix entry +Filters can be passed to the matrix as an array of strings, each matching the format of `=`. When a matrix entry does not contain the specified key, it will default to a value of empty string for regex parsing. This can be used to specify filters for keys that don't exist or keys that optionally exist and match a regex, as seen in the below example. @@ -394,6 +397,71 @@ named "ExcludedKey", a framework variable containing either "461" or "5.0", and -Filters @("ExcludedKey=^$", "framework=(461|5\.0)", "SupportedClouds=^$|.*Public.*") ``` +#### Replace values + +Replacements for values can be passed to the matrix as an array of strings, each matching the format of `=/`. +The replace argument will find any permutations where the key fully matches the key regex and the value fully matches the value regex, and replace the value with +the replacement specified. + +NOTE: +- The replacement value supports regex capture groups, enabling substring transformations, e.g. `Foo=(.*)-replaceMe/$1-replaced`. See the below examples for usage. +- For each key/value, the first replacement provided that matches will be the only one applied. +- If `=` or `/` characters need to be part of the regex or replacement, escape them with `\`. + +For example, given a matrix config like below: + +``` +{ + "matrix": { + "Agent": { + "ubuntu-1804": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" } + }, + "JavaTestVersion": [ "1.8", "1.11" ] + } +} + +``` + +The normal matrix output (without replacements), looks like: + +``` +$ ./Create-JobMatrix.ps1 -ConfigPath -Selection all +{ + "ubuntu1804_18": { + "OSVmImage": "MMSUbuntu18.04", + "Pool": "azsdk-pool-mms-ubuntu-1804-general", + "JavaTestVersion": "1.8" + }, + "ubuntu1804_111": { + "OSVmImage": "MMSUbuntu18.04", + "Pool": "azsdk-pool-mms-ubuntu-1804-general", + "JavaTestVersion": "1.11" + } +} +``` + +Passing in multiple replacements, the output will look like below. Note that replacing key/values that appear nested within a grouping +will not affect that segment of the job name, since the job takes the grouping name (in this case "ubuntu1804"). + +The below example includes samples of regex grouping references, and wildcard key/value regexes: + +``` +$ $replacements = @('.*Version=1.11/2.0', 'Pool=(.*ubuntu.*)-general/$1-custom') +$ ../Create-JobMatrix.ps1 -ConfigPath ./test.Json -Selection all -Replace $replacements +{ + "ubuntu1804_18": { + "OSVmImage": "MMSUbuntu18.04", + "Pool": "azsdk-pool-mms-ubuntu-1804-custom", + "JavaTestVersion": "1.8" + }, + "ubuntu1804_20": { + "OSVmImage": "MMSUbuntu18.04", + "Pool": "azsdk-pool-mms-ubuntu-1804-custom", + "JavaTestVersion": "2.0" + } +} +``` + #### NonSparseParameters Sometimes it may be necessary to generate a sparse matrix, but keep the full combination of a few parameters. The diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 78dbc7417..d801c0527 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -4,39 +4,98 @@ class MatrixConfig { [PSCustomObject]$displayNames [Hashtable]$displayNamesLookup [PSCustomObject]$matrix - [System.Collections.Specialized.OrderedDictionary]$orderedMatrix + [MatrixParameter[]]$matrixParameters [Array]$include [Array]$exclude } -$IMPORT_KEYWORD = '$IMPORT' - -function CreateDisplayName([string]$parameter, [Hashtable]$displayNamesLookup) -{ - $name = $parameter.ToString() - - if ($displayNamesLookup.ContainsKey($parameter)) { - $name = $displayNamesLookup[$parameter] +class MatrixParameter { + MatrixParameter([String]$name, [System.Object]$value) { + $this.Value = $value + $this.Name = $name } - # Matrix naming restrictions: - # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration - $name = $name -replace "[^A-Za-z0-9_]", "" - return $name + [System.Object]$Value + [System.Object]$Name + + Set($value, [String]$keyRegex = '') + { + if ($this.Value -is [PSCustomObject]) { + $set = $false + foreach ($prop in $this.Value.PSObject.Properties) { + if ($prop.Name -match $keyRegex) { + $prop.Value = $value + $set = $true + break + } + } + if (!$set) { + throw "Property `"$keyRegex`" does not exist for MatrixParameter." + } + } else { + $this.Value = $value + } + } + + [System.Object]Flatten() + { + if ($this.Value -is [PSCustomObject]) { + return $this.Value.PSObject.Properties | ForEach-Object { + [MatrixParameter]::new($_.Name, $_.Value) + } + } elseif ($this.Value -is [Array]) { + return $this.Value | ForEach-Object { + [MatrixParameter]::new($this.Name, $_) + } + } else { + return $this + } + } + + [Int]Length() + { + if ($this.Value -is [PSCustomObject]) { + return ($this.Value.PSObject.Properties | Measure-Object).Count + } elseif ($this.Value -is [Array]) { + return $this.Value.Length + } else { + return 1 + } + } + + [String]CreateDisplayName([Hashtable]$displayNamesLookup) + { + $displayName = $this.Value.ToString() + if ($this.Value -is [PSCustomObject]) { + $displayName = $this.Name + } + + if ($displayNamesLookup.ContainsKey($displayName)) { + $displayName = $displayNamesLookup[$displayName] + } + + # Matrix naming restrictions: + # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration + $displayName = $displayName -replace "[^A-Za-z0-9_]", "" + return $displayName + } } +$IMPORT_KEYWORD = '$IMPORT' + function GenerateMatrix( [MatrixConfig]$config, [String]$selectFromMatrixType, [String]$displayNameFilter = ".*", [Array]$filters = @(), + [Array]$replace = @(), [Array]$nonSparseParameters = @() ) { - $orderedMatrix, $importedMatrix, $importedDisplayNamesLookup = ProcessImport $config.orderedMatrix $selectFromMatrixType + $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = ProcessImport $config.matrixParameters $selectFromMatrixType $config.displayNamesLookup if ($selectFromMatrixType -eq "sparse") { - [Array]$matrix = GenerateSparseMatrix $orderedMatrix $config.displayNamesLookup $nonSparseParameters + $matrix = GenerateSparseMatrix $matrixParameters $config.displayNamesLookup $nonSparseParameters } elseif ($selectFromMatrixType -eq "all") { - [Array]$matrix = GenerateFullMatrix $orderedMatrix $config.displayNamesLookup + $matrix = GenerateFullMatrix $matrixParameters $config.displayNamesLookup } else { throw "Matrix generator not implemented for selectFromMatrixType: $($platform.selectFromMatrixType)" } @@ -44,37 +103,37 @@ function GenerateMatrix( # Combine with imported after matrix generation, since a sparse selection should result in a full combination of the # top level and imported sparse matrices (as opposed to a sparse selection of both matrices). if ($importedMatrix) { - [Array]$matrix = CombineMatrices $matrix $importedMatrix $importedDisplayNamesLookup + $matrix = CombineMatrices $matrix $importedMatrix $combinedDisplayNameLookup } - if ($config.exclude) { - [Array]$matrix = ProcessExcludes $matrix $config.exclude + $matrix = ProcessExcludes $matrix $config.exclude } if ($config.include) { - [Array]$matrix = ProcessIncludes $config $matrix $selectFromMatrixType + $matrix = ProcessIncludes $config $matrix $selectFromMatrixType } - [Array]$matrix = FilterMatrixDisplayName $matrix $displayNameFilter - [Array]$matrix = FilterMatrix $matrix $filters + $matrix = FilterMatrix $matrix $filters + $matrix = ProcessReplace $matrix $replace $config.displayNamesLookup + $matrix = FilterMatrixDisplayName $matrix $displayNameFilter return $matrix } function ProcessNonSparseParameters( - [System.Collections.Specialized.OrderedDictionary]$parameters, + [MatrixParameter[]]$parameters, [Array]$nonSparseParameters ) { if (!$nonSparseParameters) { return $parameters, $null } - $sparse = [ordered]@{} - $nonSparse = [ordered]@{} + $sparse = [MatrixParameter[]]@() + $nonSparse = [MatrixParameter[]]@() - foreach ($param in $parameters.GetEnumerator()) { + foreach ($param in $parameters) { if ($param.Name -in $nonSparseParameters) { - $nonSparse[$param.Name] = $param.Value + $nonSparse += $param } else { - $sparse[$param.Name] = $param.Value + $sparse += $param } } @@ -82,7 +141,7 @@ function ProcessNonSparseParameters( } function FilterMatrixDisplayName([array]$matrix, [string]$filter) { - return $matrix | ForEach-Object { + return $matrix | Where-Object { $_ } | ForEach-Object { if ($_.Name -match $filter) { return $_ } @@ -132,37 +191,42 @@ function ParseFilter([string]$filter) { function GetMatrixConfigFromJson([String]$jsonConfig) { [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json - $config.orderedMatrix = [ordered]@{} + $config.matrixParameters = @() $config.displayNamesLookup = @{} + $include = [MatrixParameter[]]@() + $exclude = [MatrixParameter[]]@() - if ($null -ne $config.matrix) { - $config.matrix.PSObject.Properties | ForEach-Object { - $config.orderedMatrix.Add($_.Name, $_.Value) - } - } if ($null -ne $config.displayNames) { $config.displayNames.PSObject.Properties | ForEach-Object { $config.displayNamesLookup.Add($_.Name, $_.Value) } } - $config.include = $config.include | Where-Object { $null -ne $_ } | ForEach-Object { - $ordered = [ordered]@{} - $_.PSObject.Properties | ForEach-Object { - $ordered.Add($_.Name, $_.Value) - } - return $ordered + if ($null -ne $config.matrix) { + $config.matrixParameters = PsObjectToMatrixParameterArray $config.matrix } - $config.exclude = $config.exclude | Where-Object { $null -ne $_ } | ForEach-Object { - $ordered = [ordered]@{} - $_.PSObject.Properties | ForEach-Object { - $ordered.Add($_.Name, $_.Value) - } - return $ordered + foreach ($includeMatrix in $config.include) { + $include += ,@(PsObjectToMatrixParameterArray $includeMatrix) + } + foreach ($excludeMatrix in $config.exclude) { + $exclude += ,@(PsObjectToMatrixParameterArray $excludeMatrix) } + $config.include = $include + $config.exclude = $exclude + return $config } +function PsObjectToMatrixParameterArray([PSCustomObject]$obj) +{ + if ($obj -eq $null) { + return $null + } + return $obj.PSObject.Properties | ForEach-Object { + [MatrixParameter]::new($_.Name, $_.Value) + } +} + function ProcessExcludes([Array]$matrix, [Array]$excludes) { $deleteKey = "%DELETE%" @@ -196,18 +260,105 @@ function ProcessIncludes([MatrixConfig]$config, [Array]$matrix) return $matrix + $inclusionMatrix } -function ProcessImport([System.Collections.Specialized.OrderedDictionary]$matrix, [String]$selection) -{ - if (!$matrix -or !$matrix.Contains($IMPORT_KEYWORD)) { - return $matrix, @(), @{} +function ParseReplacement([String]$replacement) { + $parsed = '', '', '' + $idx = 0 + $escaped = $false + $operators = '=', '/' + $err = "Invalid replacement syntax, expecting =/" + + foreach ($c in $replacement -split '') { + if ($idx -ge $parsed.Length) { + throw $err + } + if (!$escaped -and $c -in $operators) { + $idx++ + } else { + $parsed[$idx] += $c + } + $escaped = $c -eq '\' } - $importPath = $matrix[$IMPORT_KEYWORD] - $matrix.Remove($IMPORT_KEYWORD) + if ($idx -lt $parsed.Length - 1) { + throw $err + } + + $replace = $parsed[2] -replace "\\([$($operators -join '')])", '$1' + + return @{ + "key" = '^' + $parsed[0] + '$' + # Force full matches only. + "value" = '^' + $parsed[1] + '$' + "replace" = $replace + } +} + +function ProcessReplace +{ + param( + [Array]$matrix, + [Array]$replacements, + [Hashtable]$displayNamesLookup + ) + + if (!$replacements) { + return $matrix + } + + $replaceMatrix = @() + + foreach ($element in $matrix) { + $replacement = [MatrixParameter[]]@() + + foreach ($perm in $element._permutation) { + $replace = $perm + + # Iterate nested permutations or run once for singular values (int, string, bool) + foreach ($flattened in $perm.Flatten()) { + foreach ($query in $replacements) { + $parsed = ParseReplacement $query + if ($flattened.Name -match $parsed.key -and $flattened.Value -match $parsed.value) { + # In most cases, this will just swap one value for another, however -replace + # is used here in order to support replace values which may use regex capture groups + # e.g. 'foo-1' -replace '(foo)-1', '$1-replaced' + $replaceValue = $flattened.Value -replace $parsed.value, $parsed.replace + $perm.Set($replaceValue, $parsed.key) + break + } + } + } + + $replacement += $perm + } + + $replaceMatrix += CreateMatrixCombinationScalar $replacement $displayNamesLookup + } + + return $replaceMatrix +} + +function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Hashtable]$displayNamesLookup) +{ + $importPath = "" + $matrix = $matrix | ForEach-Object { + if ($_.Name -ne $IMPORT_KEYWORD) { + return $_ + } else { + $importPath = $_.Value + } + } + if (!$matrix -or !$importPath) { + return $matrix, @() + } $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath) $importedMatrix = GenerateMatrix $importedMatrixConfig $selection + $combinedDisplayNameLookup = $importedMatrixConfig.displayNamesLookup + foreach ($lookup in $displayNamesLookup.GetEnumerator()) { + $combinedDisplayNameLookup[$lookup.Name] = $lookup.Value + } + return $matrix, $importedMatrix, $importedMatrixConfig.displayNamesLookup } @@ -223,27 +374,7 @@ function CombineMatrices([Array]$matrix1, [Array]$matrix2, [Hashtable]$displayNa foreach ($entry1 in $matrix1) { foreach ($entry2 in $matrix2) { - $entry2name = @() - $newEntry = @{ - name = $entry1.name - parameters = CloneOrderedDictionary $entry1.parameters - } - foreach($param in $entry2.parameters.GetEnumerator()) { - if (!$newEntry.parameters.Contains($param.Name)) { - $newEntry.parameters[$param.Name] = $param.Value - $entry2name += CreateDisplayName $param.Value $displayNamesLookup - } else { - Write-Warning "Skipping duplicate parameter `"$($param.Name)`" when combining matrix." - } - } - - # The maximum allowed matrix name length is 100 characters - $newEntry.name = @($newEntry.name, ($entry2name -join "_")) -join "_" - if ($newEntry.name.Length -gt 100) { - $newEntry.name = $newEntry.name[0..99] -join "" - } - - $combined += $newEntry + $combined += CreateMatrixCombinationScalar ($entry1._permutation + $entry2._permutation) $displayNamesLookup } } @@ -277,6 +408,10 @@ function SerializePipelineMatrix([Array]$matrix) { $pipelineMatrix = [Ordered]@{} foreach ($entry in $matrix) { + if ($pipelineMatrix.Contains($entry.Name)) { + Write-Warning "Found duplicate configurations for job `"$($entry.name)`". Multiple values may have been replaced with the same value." + continue + } $pipelineMatrix.Add($entry.name, [Ordered]@{}) foreach ($key in $entry.parameters.Keys) { $pipelineMatrix[$entry.name].Add($key, $entry.parameters[$key]) @@ -290,13 +425,13 @@ function SerializePipelineMatrix([Array]$matrix) } function GenerateSparseMatrix( - [System.Collections.Specialized.OrderedDictionary]$parameters, + [MatrixParameter[]]$parameters, [Hashtable]$displayNamesLookup, [Array]$nonSparseParameters = @() ) { $parameters, $nonSparse = ProcessNonSparseParameters $parameters $nonSparseParameters - [Array]$dimensions = GetMatrixDimensions $parameters - [Array]$matrix = GenerateFullMatrix $parameters $displayNamesLookup + $dimensions = GetMatrixDimensions $parameters + $matrix = GenerateFullMatrix $parameters $displayNamesLookup $sparseMatrix = @() $indexes = GetSparseMatrixIndexes $dimensions @@ -305,7 +440,7 @@ function GenerateSparseMatrix( } if ($nonSparse) { - [Array]$allOfMatrix = GenerateFullMatrix $nonSparse $displayNamesLookup + $allOfMatrix = GenerateFullMatrix $nonSparse $displayNamesLookup return CombineMatrices $allOfMatrix $sparseMatrix $displayNamesLookup } @@ -335,7 +470,7 @@ function GetSparseMatrixIndexes([Array]$dimensions) } function GenerateFullMatrix( - [System.Collections.Specialized.OrderedDictionary] $parameters, + [MatrixParameter[]] $parameters, [Hashtable]$displayNamesLookup = @{} ) { # Handle when the config does not have a matrix specified (e.g. only the include field is specified) @@ -343,32 +478,29 @@ function GenerateFullMatrix( return @() } - $parameterArray = $parameters.GetEnumerator() | ForEach-Object { $_ } - $matrix = [System.Collections.ArrayList]::new() - InitializeMatrix $parameterArray $displayNamesLookup $matrix + InitializeMatrix $parameters $displayNamesLookup $matrix return $matrix } -function CreateMatrixEntry([System.Collections.Specialized.OrderedDictionary]$permutation, [Hashtable]$displayNamesLookup = @{}) +function CreateMatrixCombinationScalar([MatrixParameter[]]$permutation, [Hashtable]$displayNamesLookup = @{}) { $names = @() - $splattedParameters = [Ordered]@{} + $flattenedParameters = [Ordered]@{} - foreach ($entry in $permutation.GetEnumerator()) { + foreach ($entry in $permutation) { $nameSegment = "" - if ($entry.Value -is [PSCustomObject]) { - $nameSegment = CreateDisplayName $entry.Name $displayNamesLookup - foreach ($toSplat in $entry.Value.PSObject.Properties) { - $splattedParameters.Add($toSplat.Name, $toSplat.Value) + # Unwind nested permutations or run once for singular values (int, string, bool) + foreach ($param in $entry.Flatten()) { + if ($flattenedParameters.Contains($param.Name)) { + throw "Found duplicate parameter `"$($param.Name)`" when creating matrix combination." } - } else { - $nameSegment = CreateDisplayName $entry.Value $displayNamesLookup - $splattedParameters.Add($entry.Name, $entry.Value) + $flattenedParameters.Add($param.Name, $param.Value) } + $nameSegment = $entry.CreateDisplayName($displayNamesLookup) if ($nameSegment) { $names += $nameSegment } @@ -388,53 +520,40 @@ function CreateMatrixEntry([System.Collections.Specialized.OrderedDictionary]$pe return @{ name = $name - parameters = $splattedParameters + parameters = $flattenedParameters + # Keep the original permutation around in case we need to re-process this entry when transforming the matrix + _permutation = $permutation } } function InitializeMatrix { param( - [Array]$parameters, + [MatrixParameter[]]$parameters, [Hashtable]$displayNamesLookup, [System.Collections.ArrayList]$permutations, - $permutation = [Ordered]@{} + $permutation = [MatrixParameter[]]@() ) $head, $tail = $parameters if (!$head) { - $entry = CreateMatrixEntry $permutation $displayNamesLookup + $entry = CreateMatrixCombinationScalar $permutation $displayNamesLookup $permutations.Add($entry) | Out-Null return } # This behavior implicitly treats non-array values as single elements - foreach ($value in $head.Value) { - $newPermutation = CloneOrderedDictionary $permutation - if ($value -is [PSCustomObject]) { - foreach ($nestedParameter in $value.PSObject.Properties) { - $nestedPermutation = CloneOrderedDictionary $newPermutation - $nestedPermutation[$nestedParameter.Name] = $nestedParameter.Value - InitializeMatrix $tail $displayNamesLookup $permutations $nestedPermutation - } - } else { - $newPermutation[$head.Name] = $value - InitializeMatrix $tail $displayNamesLookup $permutations $newPermutation - } + foreach ($param in $head.Flatten()) { + $newPermutation = $permutation + $param + InitializeMatrix $tail $displayNamesLookup $permutations $newPermutation } } -function GetMatrixDimensions([System.Collections.Specialized.OrderedDictionary]$parameters) +function GetMatrixDimensions([MatrixParameter[]]$parameters) { $dimensions = @() - foreach ($param in $parameters.GetEnumerator()) { - if ($param.Value -is [PSCustomObject]) { - $dimensions += ($param.Value.PSObject.Properties | Measure-Object).Count - } elseif ($param.Value -is [Array]) { - $dimensions += $param.Value.Length - } else { - $dimensions += 1 - } + foreach ($param in $parameters) { + $dimensions += $param.Length() } return $dimensions diff --git a/eng/common/scripts/job-matrix/samples/matrix-test.yml b/eng/common/scripts/job-matrix/samples/matrix-test.yml index bead42609..4006e05f7 100644 --- a/eng/common/scripts/job-matrix/samples/matrix-test.yml +++ b/eng/common/scripts/job-matrix/samples/matrix-test.yml @@ -14,7 +14,12 @@ jobs: SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) Location: eastus2 Cloud: Public - MatrixFilters: [] + MatrixFilters: + # Exclusion example + - OSVmImage=^(?!macOS).* + MatrixReplace: + - OsVmImage=.*ubuntu.*/ubuntu-20.04 + - .*Framework.*=net5.0/net5.1 MatrixConfigs: - Name: base_product_matrix Path: eng/common/scripts/job-matrix/samples/matrix.json diff --git a/eng/common/scripts/job-matrix/samples/matrix.json b/eng/common/scripts/job-matrix/samples/matrix.json index cf88ef027..a9e291604 100644 --- a/eng/common/scripts/job-matrix/samples/matrix.json +++ b/eng/common/scripts/job-matrix/samples/matrix.json @@ -4,16 +4,16 @@ }, "matrix": { "Agent": { - "ubuntu-18.04": { "OSVmImage": "ubuntu-18.04", "Pool": "Azure Pipelines" }, - "windows-2019": { "OSVmImage": "windows-2019", "Pool": "Azure Pipelines" }, - "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } + "ubuntu": { "OSVmImage": "ubuntu-18.04", "Pool": "Azure Pipelines" }, + "windows": { "OSVmImage": "windows-2019", "Pool": "Azure Pipelines" }, + "macOS": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } }, "TestTargetFramework": [ "netcoreapp2.1", "net461", "net5.0" ] }, "include": [ { "Agent": { - "windows-2019": { "OSVmImage": "windows-2019", "Pool": "Azure Pipelines" } + "windows": { "OSVmImage": "windows-2019", "Pool": "Azure Pipelines" } }, "TestTargetFramework": [ "net461", "net5.0" ], "AdditionalTestArguments": "/p:UseProjectReferenceToAzureClients=true" diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 index c7583e067..cd721c351 100644 --- a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 @@ -9,9 +9,9 @@ BeforeAll { for ($i = 0; $i -lt $matrix.Length; $i++) { foreach ($entry in $matrix[$i]) { - $expected[$i].name | Should -Be $entry.name + $entry.name | Should -Be $expected[$i].name foreach ($param in $entry.parameters.GetEnumerator()) { - $expected[$i].parameters[$param.Name] | Should -Be $param.Value + $param.Value | Should -Be $expected[$i].parameters[$param.Name] } } } @@ -33,18 +33,25 @@ Describe "Platform Matrix nonSparse" -Tag "nonsparse" { } It "Should process nonSparse parameters" { - $parameters, $nonSparse = ProcessNonSparseParameters $config.orderedMatrix "testField1","testField3" - $parameters.Count | Should -Be 1 - $parameters["testField2"] | Should -Be 1,2,3 - $nonSparse.Count | Should -Be 2 - $nonSparse["testField1"] | Should -Be 1,2 - $nonSparse["testField3"] | Should -Be 1,2,3,4 + $parameters, $nonSparse = ProcessNonSparseParameters $config.matrixParameters "testField1","testField3" - $parameters, $nonSparse = ProcessNonSparseParameters $config.orderedMatrix "testField3" + $parameters.Count | Should -Be 1 + $parameters[0].Name | Should -Be "testField2" + $parameters[0].Value | Should -Be 1,2,3 + + $nonSparse.Count | Should -Be 2 + $nonSparse[0].Name | Should -Be "testField1" + $nonSparse[0].Value | Should -Be 1,2 + $nonSparse[1].Name | Should -Be "testField3" + $nonSparse[1].Value | Should -Be 1,2,3,4 + + $parameters, $nonSparse = ProcessNonSparseParameters $config.matrixParameters "testField3" $parameters.Count | Should -Be 2 - $parameters.Contains("testField3") | Should -Be $false + ($parameters).Name -match "testField3" | Should -Be $null + $nonSparse.Count | Should -Be 1 - $nonSparse["testField3"] | Should -Be 1,2,3,4 + $nonSparse[0].Name | Should -Be "testField3" + $nonSparse[0].Value | Should -Be 1,2,3,4 } It "Should ignore nonSparse with all selection" { @@ -96,7 +103,7 @@ Describe "Platform Matrix Import" -Tag "import" { $matrix[0].name | Should -Be test1_foo1_bar1 $matrix[0].parameters.testField | Should -Be "test1" $matrix[0].parameters.Foo | Should -Be "foo1" - $matrix[2].name | Should -Be test1_importedBaz + $matrix[2].name | Should -Be test1_importedBazName $matrix[2].parameters.testField | Should -Be "test1" $matrix[2].parameters.Baz | Should -Be "importedBaz" $matrix[4].name | Should -Be test2_foo2_bar2 @@ -104,7 +111,27 @@ Describe "Platform Matrix Import" -Tag "import" { $matrix[4].parameters.Foo | Should -Be "foo2" } - It "Should generate a sparse matrix with an imported a sparse matrix" { + It "Should source imported display name lookups" { + $matrixJson = @' +{ + "displayNames": { + "test1": "test1DisplayName", + "importedBaz": "importedBazNameOverride" + }, + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField": [ "test1", "test2" ] + } +} +'@ + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" -nonSparseParameters "testField" + + $matrix[0].name | Should -Be test1DisplayName_foo1_bar1 + $matrix[2].name | Should -Be test1DisplayName_importedBazNameOverride + } + + It "Should generate a sparse matrix with an imported sparse matrix" { $matrixJson = @' { "matrix": { @@ -127,7 +154,7 @@ Describe "Platform Matrix Import" -Tag "import" { }, { "parameters": { "testField1": "test11", "testField2": "test21", "Baz": "importedBaz" }, - "name": "test11_test21_importedBaz" + "name": "test11_test21_importedBazName" }, { "parameters": { "testField1": "test12", "testField2": "test22", "Foo": "foo1", "Bar": "bar1" }, @@ -139,7 +166,7 @@ Describe "Platform Matrix Import" -Tag "import" { }, { "parameters": { "testField1": "test12", "testField2": "test22", "Baz": "importedBaz" }, - "name": "test12_test22_importedBaz" + "name": "test12_test22_importedBazName" } ] '@ @@ -188,7 +215,7 @@ Describe "Platform Matrix Import" -Tag "import" { }, { "parameters": { "testField": "test2", "Baz": "importedBaz" }, - "name": "test2_importedBaz" + "name": "test2_importedBazName" }, { "parameters": { "testField": "test3", "Foo": "foo1", "Bar": "bar1" }, @@ -217,7 +244,7 @@ Describe "Platform Matrix Import" -Tag "import" { CompareMatrices $matrix $expected } - It "Should generate a sparse matrix with an imported a sparse matrix" { + It "Should not combine matrices with duplicate keys" { $matrixJson = @' { "matrix": { @@ -228,13 +255,153 @@ Describe "Platform Matrix Import" -Tag "import" { '@ $importConfig = GetMatrixConfigFromJson $matrixJson - $matrix = GenerateMatrix $importConfig "sparse" + { GenerateMatrix $importConfig "sparse" } | Should -Throw + } - $matrix[0].parameters["Foo"] | Should -Be "fooOverride1" - $matrix[0].name | Should -Be "fooOverride1_bar1" - $matrix[3].parameters["Foo"] | Should -Be "fooOverride2" - $matrix[3].name | Should -Be "fooOverride2_bar1" - $matrix[5].parameters["Foo"] | Should -Be "fooOverride2" - $matrix[5].name | Should -Be "fooOverride2_importedBaz" +} + +Describe "Platform Matrix Replace" -Tag "replace" { + It "Should parse replacement syntax" -TestCases @( + @{ query = 'foo=bar/baz'; key = '^foo$'; value = '^bar$'; replace = 'baz' }, + @{ query = 'foo=\/p:bar/\/p:baz'; key = '^foo$'; value = '^\/p:bar$'; replace = '/p:baz' }, + @{ query = 'f\=o\/o=\/p:b\=ar/\/p:b\=az'; key = '^f\=o\/o$'; value = '^\/p:b\=ar$'; replace = '/p:b=az' }, + @{ query = 'foo=bar/'; key = '^foo$'; value = '^bar$'; replace = '' }, + @{ query = 'foo=/baz'; key = '^foo$'; value = '^$'; replace = 'baz' } + ) { + $parsed = ParseReplacement $query + $parsed.key | Should -Be $key + $parsed.value | Should -Be $value + $parsed.replace | Should -Be $replace + } + + It "Should fail for invalid replacement syntax" -TestCases @( + @{ query = '' }, + @{ query = 'asdf' }, + @{ query = 'asdf=foo/bar/baz' }, + @{ query = 'asdf=foo=bar/baz' }, + @{ query = 'asdf=foo' } + ) { + { $parsed = ParseReplacement $query } | Should -Throw + { $parsed = ParseReplacement $query } | Should -Throw + { $parsed = ParseReplacement $query } | Should -Throw + { $parsed = ParseReplacement $query } | Should -Throw + { $parsed = ParseReplacement $query } | Should -Throw + } + + It "Should replace values in a matrix" { + $matrixJson = @' +{ + "matrix": { + "Foo": [ "foo1", "foo2" ], + "Bar": [ "bar1", "bar2" ] + }, + "include": [ { "Baz": "baz1" } ] +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "Foo": "foo1Replaced", "Bar": "bar1" }, + "name": "foo1Replaced_bar1" + }, + { + "parameters": { "Foo": "fooDefaultReplaced", "Bar": "bar2" }, + "name": "fooDefaultReplaced_bar2" + }, + { + "parameters": { "Baz": "bazReplaced" }, + "name": "bazReplaced" + } +] +'@ + + $replace = @( + "Foo=foo1/foo1Replaced", + "Foo=foo.*/fooDefaultReplaced", + ".*=B.z\d/bazReplaced" + ) + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix -config $importConfig -selectFromMatrixType "sparse" -replace $replace + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 3 + CompareMatrices $matrix $expected + } + + It "Should replace values in a matrix with import and nonSparseParameters" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField": [ "test1", "test2" ] + } +} +'@ + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" -nonSparseParameters "testField" -replace @("testField=test1/testReplaced", "Baz=.*/bazReplaced") + + $matrix.Length | Should -Be 6 + + $matrix[0].name | Should -Be testReplaced_foo1_bar1 + $matrix[0].parameters.testField | Should -Be "testReplaced" + $matrix[0].parameters.Foo | Should -Be "foo1" + $matrix[2].name | Should -Be testReplaced_bazReplaced + $matrix[2].parameters.testField | Should -Be "testReplaced" + $matrix[2].parameters.Baz | Should -Be "bazReplaced" + $matrix[4].name | Should -Be test2_foo2_bar2 + $matrix[4].parameters.testField | Should -Be "test2" + $matrix[4].parameters.Foo | Should -Be "foo2" + } + + It "Should replace values in groupings" { + $matrixJson = @' +{ + "matrix": { + "Agent": { + "ubuntu-1804": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" } + }, + "JavaTestVersion": [ "1.8", "1.11" ] + } +} +'@ + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "all" -replace @("JavaTestVersion=1.8/2.0", "Pool=.*ubuntu.*/custom-ubuntu-pool") + + $matrix.Length | Should -Be 2 + # Replacements of inner values will preserve the grouping name + $matrix[0].name | Should -Be "ubuntu1804_20" + $matrix[0].parameters.JavaTestVersion | Should -Be "2.0" + $matrix[0].parameters.Pool | Should -Be "custom-ubuntu-pool" + $matrix[0].parameters.OSVmImage | Should -Be "MMSUbuntu18.04" + + # Make sure non-literal keys still replace under the hood + $matrix = GenerateMatrix $importConfig "all" -replace ".*=.*ubuntu.*/custom-ubuntu-pool" + + $matrix.Length | Should -Be 2 + $matrix[0].name | Should -Be "ubuntu1804_18" + $matrix[0].parameters.Pool | Should -Be "custom-ubuntu-pool" + } + + It "Should replace values and apply regex capture groups" { + $matrixJson = @' +{ + "matrix": { + "Foo": [ "foo1", "foo2" ], + "Bar": [ "bar1", "bar2" ] + } +} +'@ + $importConfig = GetMatrixConfigFromJson $matrixJson + $replace = 'Foo=(foo)1/$1ReplacedFoo1', 'B.*=(.*)2/$1ReplacedBar2' + $matrix = GenerateMatrix $importConfig "sparse" -replace $replace + + $matrix.Length | Should -Be 2 + $matrix[0].name | Should -Be "fooReplacedFoo1_bar1" + $matrix[0].parameters.Foo | Should -Be "fooReplacedFoo1" + + $matrix[1].name | Should -Be "foo2_barReplacedBar2" + $matrix[1].parameters.Bar | Should -Be "barReplacedBar2" } } diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 index ab9ce82a0..500d3d363 100644 --- a/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 @@ -307,15 +307,12 @@ Describe "Platform Matrix Generation" -Tag "generate" { } It "Should get matrix dimensions from Nd parameters" { - GetMatrixDimensions $generateConfig.orderedMatrix | Should -Be 3, 2, 2 - - $generateConfig.orderedMatrix.Add("testStringParameter", "test") - GetMatrixDimensions $generateConfig.orderedMatrix | Should -Be 3, 2, 2, 1 + GetMatrixDimensions $generateConfig.matrixParameters | Should -Be 3, 2, 2 } It "Should use name overrides from displayNames" { - $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix - $matrix = GenerateFullMatrix $generateConfig.orderedMatrix $generateconfig.displayNamesLookup + $dimensions = GetMatrixDimensions $generateConfig.matrixParameters + $matrix = GenerateFullMatrix $generateConfig.matrixParameters $generateconfig.displayNamesLookup $element = GetNdMatrixElement @(0, 0, 0) $matrix $dimensions $element.name | Should -Be "windows2019_net461" @@ -330,8 +327,8 @@ Describe "Platform Matrix Generation" -Tag "generate" { It "Should enforce valid display name format" { $generateconfig.displayNamesLookup["net461"] = '123.Some.456.Invalid_format-name$(foo)' $generateconfig.displayNamesLookup["netcoreapp2.1"] = (New-Object string[] 150) -join "a" - $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix - $matrix = GenerateFullMatrix $generateconfig.orderedMatrix $generateconfig.displayNamesLookup + $dimensions = GetMatrixDimensions $generateConfig.matrixParameters + $matrix = GenerateFullMatrix $generateconfig.matrixParameters $generateconfig.displayNamesLookup $element = GetNdMatrixElement @(0, 0, 0) $matrix $dimensions $element.name | Should -Be "windows2019_123some456invalid_formatnamefoo" @@ -344,8 +341,8 @@ Describe "Platform Matrix Generation" -Tag "generate" { It "Should initialize an N-dimensional matrix from all parameter permutations" { - $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix - $matrix = GenerateFullMatrix $generateConfig.orderedMatrix $generateConfig.displayNamesLookup + $dimensions = GetMatrixDimensions $generateConfig.matrixParameters + $matrix = GenerateFullMatrix $generateConfig.matrixParameters $generateConfig.displayNamesLookup $matrix.Count | Should -Be 12 $element = $matrix[0].parameters @@ -369,8 +366,8 @@ Describe "Platform Matrix Generation" -Tag "generate" { @{ i = 1; name = "ubuntu1804_netcoreapp21_withfoo"; operatingSystem = "ubuntu-18.04"; framework = "netcoreapp2.1"; additionalArguments = "--enableFoo"; } @{ i = 2; name = "macOS1015_net461"; operatingSystem = "macOS-10.15"; framework = "net461"; additionalArguments = ""; } ) { - $sparseMatrix = GenerateSparseMatrix $generateConfig.orderedMatrix $generateConfig.displayNamesLookup - $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $sparseMatrix = GenerateSparseMatrix $generateConfig.matrixParameters $generateConfig.displayNamesLookup + $dimensions = GetMatrixDimensions $generateConfig.matrixParameters $size = ($dimensions | Measure-Object -Maximum).Maximum $sparseMatrix.Count | Should -Be $size @@ -398,18 +395,14 @@ Describe "Config File Object Conversion" -Tag "convert" { } It "Should convert a matrix config" { - $config.orderedMatrix | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] - $config.orderedMatrix.operatingSystem[0] | Should -Be "windows-2019" + $config.matrixParameters[0].Name | Should -Be "operatingSystem" + $config.matrixParameters[0].Flatten()[0].Value | Should -Be "windows-2019" $config.displayNamesLookup | Should -BeOfType [Hashtable] $config.displayNamesLookup["--enableFoo"] | Should -Be "withFoo" - $config.include | ForEach-Object { - $_ | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] - } - $config.exclude | ForEach-Object { - $_ | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] - } + $config.include.Length | Should -Be 1 + $config.exclude.Length | Should -Be 3 } } @@ -429,17 +422,17 @@ Describe "Platform Matrix Post Transformation" -Tag "transform" { } It "Should remove matrix elements based on exclude filters" { - $matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup + $matrix = GenerateFullMatrix $config.matrixParameters $config.displayNamesLookup $withExclusion = ProcessExcludes $matrix $config.exclude $withExclusion.Length | Should -Be 5 - $matrix = GenerateSparseMatrix $config.orderedMatrix $config.displayNamesLookup + $matrix = GenerateSparseMatrix $config.matrixParameters $config.displayNamesLookup [array]$withExclusion = ProcessExcludes $matrix $config.exclude $withExclusion.Length | Should -Be 1 } It "Should add matrix elements based on include elements" { - $matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup + $matrix = GenerateFullMatrix $config.matrixParameters $config.displayNamesLookup $withInclusion = ProcessIncludes $config $matrix "all" $withInclusion.Length | Should -Be 15 } @@ -503,7 +496,7 @@ Describe "Platform Matrix Generation With Object Fields" -Tag "objectfields" { } It "Should parse dimensions properly" { - [Array]$dimensions = GetMatrixDimensions $objectFieldConfig.orderedMatrix + [Array]$dimensions = GetMatrixDimensions $objectFieldConfig.matrixParameters $dimensions.Length | Should -Be 3 $dimensions[0] | Should -Be 2 $dimensions[1] | Should -Be 1 diff --git a/eng/common/scripts/job-matrix/tests/test-import-matrix.json b/eng/common/scripts/job-matrix/tests/test-import-matrix.json index 2048bd390..97f6c3458 100644 --- a/eng/common/scripts/job-matrix/tests/test-import-matrix.json +++ b/eng/common/scripts/job-matrix/tests/test-import-matrix.json @@ -1,4 +1,7 @@ { + "displayNames": { + "importedBaz": "importedBazName" + }, "matrix": { "Foo": [ "foo1", "foo2" ], "Bar": [ "bar1", "bar2" ]