From c2ddfee99973da16a1f6d3596728073ba275c67e Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Fri, 14 Feb 2025 11:16:59 -0800 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 9836 (#6422) * adjustments to iterating across matrix configs for packages that belong to multiple matrices --------- Co-authored-by: Scott Beddall --- .../scripts/job-matrix/Create-PrJobMatrix.ps1 | 164 +++++++++--------- 1 file changed, 85 insertions(+), 79 deletions(-) diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 7ddc70df9..27dde5430 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -94,92 +94,75 @@ function GeneratePRMatrixForBatch { $matrixBatchesByConfig = Group-ByObjectKey $Packages "CIMatrixConfigs" foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) { + # recall that while we have grouped the package info by the matrix config object, each package still has knowledge about + # every other matrix that it belongs to. + # so we actually need to get a valid matrix config object from the first package in the batch, but we need to be certain + # that the matrix config object we get is the SAME ONE that we are iterating through $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey] - $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs - + $allPossibleMatrixConfigsForFirstPackage = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs + $matrixConfig = $allPossibleMatrixConfigsForFirstPackage | Where-Object { (Get-ObjectKey $_) -eq $matrixBatchKey } $matrixResults = @() - foreach ($matrixConfig in $matrixConfigs) { - Write-Host "Generating config for $($matrixConfig.Path)" - $nonSparse = $matrixConfig.PSObject.Properties['NonSparseParameters'] ? $matrixConfig.NonSparseParameters : @() - $matrixResults = @() - if ($directBatch) { - $matrixResults = GenerateMatrixForConfig ` - -ConfigPath $matrixConfig.Path ` - -Selection $matrixConfig.Selection ` - -DisplayNameFilter $DisplayNameFilter ` - -Filters $Filters ` - -Replace $Replace ` - -NonSparseParameters $nonSparse + if (!$matrixConfig) { + Write-Error "Unable to find matrix config for $matrixBatchKey. Check the package properties for the package $($matrixBatch[0].ArtifactName)." + exit 1 + } - if ($matrixResults) { - Write-Host "We have the following direct matrix results: " - Write-Host ($matrixResults | Out-String) - } + Write-Host "Generating config for $($matrixConfig.Path)" + $nonSparse = $matrixConfig.PSObject.Properties['NonSparseParameters'] ? $matrixConfig.NonSparseParameters : @() + + if ($directBatch) { + $matrixResults = GenerateMatrixForConfig ` + -ConfigPath $matrixConfig.Path ` + -Selection $matrixConfig.Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace ` + -NonSparseParameters $nonSparse + + if ($matrixResults) { + Write-Host "We have the following direct matrix results: " + Write-Host ($matrixResults | Out-String) + } + } + else { + $matrixResults = GenerateMatrixForConfig ` + -ConfigPath $matrixConfig.Path ` + -Selection $matrixConfig.Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters ($Filters + $IndirectFilters) ` + -Replace $Replace ` + -NonSparseParameters $nonSparse + + if ($matrixResults) { + Write-Host "We have the following indirect matrix results: " + Write-Host ($matrixResults | Out-String) } else { - $matrixResults = GenerateMatrixForConfig ` - -ConfigPath $matrixConfig.Path ` - -Selection $matrixConfig.Selection ` - -DisplayNameFilter $DisplayNameFilter ` - -Filters ($Filters + $IndirectFilters) ` - -Replace $Replace ` - -NonSparseParameters $nonSparse - - if ($matrixResults) { - Write-Host "We have the following indirect matrix results: " - Write-Host ($matrixResults | Out-String) - } - else { - Write-Host "No indirect matrix results found for $($matrixConfig.Path)" - continue - } + Write-Host "No indirect matrix results found for $($matrixConfig.Path)" + continue } + } - $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE + $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE - # we only need to modify the generated job name if there is more than one matrix config + batch - $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1 + # we only need to modify the generated job name if there is more than one matrix config + batch + $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1 - # if we are doing direct packages (or a full indirect matrix), we need to walk the batches and duplicate the matrix config for each batch, fully assigning - # the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages - if ($directBatch -or $FullSparseMatrix) { - $batchSuffixNecessary = $packageBatches.Length -gt $($directBatch ? 1 : 0) - $batchCounter = 1 + # if we are doing direct packages (or a full indirect matrix), we need to walk the batches and duplicate the matrix config for each batch, fully assigning + # the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages + if ($directBatch -or $FullSparseMatrix) { + $batchSuffixNecessary = $packageBatches.Length -gt $($directBatch ? 1 : 0) + $batchCounter = 1 - foreach ($batch in $packageBatches) { - $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "," - - foreach ($matrixOutputItem in $matrixResults) { - # we need to clone this, as each item is an object with possible children - $outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable - # we just need to iterate across them, grab the parameters hashtable, and add the new key - # if there is more than one batch, we will need to add a suffix including the batch name to the job name - $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch - - if ($matrixSuffixNecessary) { - $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name - } - - if ($batchSuffixNecessary) { - $outputItem["name"] = $outputItem["name"] + "$batchNamePrefix$batchCounter" - } - - $OverallResult += $outputItem - } - $batchCounter += 1 - } - } - # in the case of indirect packages, instead of walking the batches and duplicating their matrix config entirely, - # we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages - # as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are - else { - $batchSuffixNecessary = $packageBatches.Length -gt 0 - $batchCounter = 1 - foreach ($batch in $packageBatches) { - $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "," - $outputItem = QueuePop -queue ([ref]$matrixResults) + foreach ($batch in $packageBatches) { + $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "," + foreach ($matrixOutputItem in $matrixResults) { + # we need to clone this, as each item is an object with possible children + $outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable + # we just need to iterate across them, grab the parameters hashtable, and add the new key + # if there is more than one batch, we will need to add a suffix including the batch name to the job name $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch if ($matrixSuffixNecessary) { @@ -187,18 +170,41 @@ function GeneratePRMatrixForBatch { } if ($batchSuffixNecessary) { - $outputItem["name"] = $outputItem["name"] + "_$batchNamePrefix$batchCounter" + $outputItem["name"] = $outputItem["name"] + "$batchNamePrefix$batchCounter" } - # now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results - # we will add the cloned version to OverallResult + $OverallResult += $outputItem - $batchCounter += 1 } + $batchCounter += 1 + } + } + # in the case of indirect packages, instead of walking the batches and duplicating their matrix config entirely, + # we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages + # as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are + else { + $batchSuffixNecessary = $packageBatches.Length -gt 0 + $batchCounter = 1 + foreach ($batch in $packageBatches) { + $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "," + $outputItem = QueuePop -queue ([ref]$matrixResults) + + $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch + + if ($matrixSuffixNecessary) { + $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name + } + + if ($batchSuffixNecessary) { + $outputItem["name"] = $outputItem["name"] + "_$batchNamePrefix$batchCounter" + } + # now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results + # we will add the cloned version to OverallResult + $OverallResult += $outputItem + $batchCounter += 1 } } } - return ,$OverallResult }