Sync eng/common directory with azure-sdk-tools for PR 3790 (#4041)

* Added yaml support for job matrix creation * autogen scenario matrix for stress test * Temporary Working State * update to default sparse * pr comments and some error handling * custom matrixfilename and ordering of generatedValues.yaml * common module import * JobMatrix write host Co-authored-by: Albert Cheng <albertcheng@microsoft.com>
2022-10-20 16:30:02 -07:00 · 2022-10-20 16:30:02 -07:00 · 90f1ff9364
commit 90f1ff9364
parent 378ee6cbd4
7 changed files with 254 additions and 60 deletions
--- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1
+++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1
@ -22,9 +22,9 @@ function Update-PSModulePathForCI()
  $modulePaths = $modulePaths.Where({ !$_.StartsWith($hostedAgentModulePath) })

  # Add any "az_" paths from the agent which is the lastest set of azure modules
-  $AzModuleCachPath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator
-  if ($AzModuleCachPath -and $env.PSModulePath -notcontains $AzModuleCachPath) {
-    $modulePaths += $AzModuleCachPath
+  $AzModuleCachePath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator
+  if ($AzModuleCachePath -and $env:PSModulePath -notcontains $AzModuleCachePath) {
+    $modulePaths += $AzModuleCachePath
  }

  $env:PSModulePath = $modulePaths -join $moduleSeperator
--- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1
@ -23,7 +23,7 @@ if (!(Test-Path $ConfigPath)) {
    Write-Error "ConfigPath '$ConfigPath' does not exist."
    exit 1
 }
-$config = GetMatrixConfigFromJson (Get-Content $ConfigPath)
+$config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw)
 # Strip empty string filters in order to be able to use azure pipelines yaml join()
 $Filters = $Filters | Where-Object { $_ }

@ -38,4 +38,7 @@ $Filters = $Filters | Where-Object { $_ }
 $serialized = SerializePipelineMatrix $matrix

 Write-Output $serialized.pretty
-Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"
+
+if ($null -ne $env:SYSTEM_TEAMPROJECTID) {
+    Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"
+}
--- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1
+++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1
@ -84,6 +84,7 @@ class MatrixParameter {
    }
 }

+. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1)
 $IMPORT_KEYWORD = '$IMPORT'

 function GenerateMatrix(
@ -146,7 +147,7 @@ function ProcessNonSparseParameters(

 function FilterMatrixDisplayName([array]$matrix, [string]$filter) {
    return $matrix | Where-Object { $_ } | ForEach-Object {
-        if ($_.Name -match $filter) {
+        if ($_.ContainsKey("Name") -and $_.Name -match $filter) {
            return $_
        }
    }
@ -168,7 +169,7 @@ function MatchesFilters([hashtable]$entry, [array]$filters) {
        # Default all regex checks to go against empty string when keys are missing.
        # This simplifies the filter syntax/interface to be regex only.
        $value = ""
-        if ($null -ne $entry -and $entry.parameters.Contains($key)) {
+        if ($null -ne $entry -and $entry.ContainsKey("parameters") -and $entry.parameters.Contains($key)) {
            $value = $entry.parameters[$key]
        }
        if ($value -notmatch $regex) {
@ -190,11 +191,34 @@ function ParseFilter([string]$filter) {
    }
 }

-# Importing the JSON as PSCustomObject preserves key ordering,
-# whereas ConvertFrom-Json -AsHashtable does not
+function GetMatrixConfigFromFile([String] $config)
+{
+    [MatrixConfig]$config = try{
+        GetMatrixConfigFromJson $config
+    } catch {
+        GetMatrixConfigFromYaml $config
+    }
+    return $config
+}
+
+function GetMatrixConfigFromYaml([String] $yamlConfig)
+{
+    Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+    # ConvertTo then from json is to make sure the nested values are in PSCustomObject
+    [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json
+    return GetMatrixConfig $config
+}
+
 function GetMatrixConfigFromJson([String]$jsonConfig)
 {
    [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json
+    return GetMatrixConfig $config
+}
+
+# Importing the JSON as PSCustomObject preserves key ordering,
+# whereas ConvertFrom-Json -AsHashtable does not
+function GetMatrixConfig([MatrixConfig]$config)
+{
    $config.matrixParameters = @()
    $config.displayNamesLookup = @{}
    $include = [MatrixParameter[]]@()
@ -359,7 +383,7 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n
        Write-Error "`$IMPORT path '$importPath' does not exist."
        exit 1
    }
-    $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath)
+    $importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath)
    $importedMatrix = GenerateMatrix `
                        -config $importedMatrixConfig `
                        -selectFromMatrixType $selection `
--- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1
+++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1
@ -24,7 +24,15 @@ param(
    [string]$Namespace,

    # Override remote stress-test-addons with local on-disk addons for development
-    [System.IO.FileInfo]$LocalAddonsPath
+    [System.IO.FileInfo]$LocalAddonsPath,
+
+    # Matrix generation parameters
+    [Parameter(Mandatory=$False)][string]$MatrixFileName,
+    [Parameter(Mandatory=$False)][string]$MatrixSelection,
+    [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$MatrixFilters,
+    [Parameter(Mandatory=$False)][array]$MatrixReplace,
+    [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters
 )

 . $PSScriptRoot/stress-test-deployment-lib.ps1
--- a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1
+++ b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1
@ -12,20 +12,42 @@ class StressTestPackageInfo {
    [string]$Deployer
 }

+. $PSScriptRoot/../job-matrix/job-matrix-functions.ps1
+. $PSScriptRoot/generate-scenario-matrix.ps1
+
 function FindStressPackages(
    [string]$directory,
    [hashtable]$filters = @{},
    [switch]$CI,
-    [string]$namespaceOverride
+    [string]$namespaceOverride,
+    [string]$MatrixSelection,
+    [Parameter(Mandatory=$False)][string]$MatrixFileName,
+    [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$MatrixFilters,
+    [Parameter(Mandatory=$False)][array]$MatrixReplace,
+    [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters
 ) {
    # Bare minimum filter for stress tests
    $filters['stressTest'] = 'true'
-
    $packages = @()
    $chartFiles = Get-ChildItem -Recurse -Filter 'Chart.yaml' $directory 
+    if (!$MatrixFileName) {
+        $MatrixFileName = '/scenarios-matrix.yaml'
+    }
    foreach ($chartFile in $chartFiles) {
        $chart = ParseChart $chartFile
        if (matchesAnnotations $chart $filters) {
+            $matrixFilePath = (Join-Path $chartFile.Directory.FullName $MatrixFileName)
+            if (Test-Path $matrixFilePath) {
+                GenerateScenarioMatrix `
+                    -matrixFilePath $matrixFilePath `
+                    -Selection $MatrixSelection `
+                    -DisplayNameFilter $MatrixDisplayNameFilter `
+                    -Filters $MatrixFilters `
+                    -Replace $MatrixReplace `
+                    -NonSparseParameters $MatrixNonSparseParameters
+            }
+
            $packages += NewStressTestPackageInfo `
                            -chart $chart `
                            -chartFile $chartFile `
@ -80,8 +102,8 @@ function NewStressTestPackageInfo(
        Namespace = $namespace.ToLower()
        Directory = $chartFile.DirectoryName
        ReleaseName = $chart.name
-        Dockerfile = $chart.annotations.dockerfile
-        DockerBuildDir = $chart.annotations.dockerbuilddir
+        Dockerfile = "dockerfile" -in $chart.annotations.keys ? $chart.annotations.dockerfile : $null
+        DockerBuildDir = "dockerbuilddir" -in $chart.annotations.keys ? $chart.annotations.dockerbuilddir : $null
    }
 }

--- a/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1
+++ b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1
@ -0,0 +1,86 @@
+param(
+    [string]$matrixFilePath,
+    [string]$Selection,
+    [Parameter(Mandatory=$False)][string]$DisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$Filters,
+    [Parameter(Mandatory=$False)][array]$Replace,
+    [Parameter(Mandatory=$False)][array]$NonSparseParameters
+)
+
+function GenerateScenarioMatrix(
+    [string]$matrixFilePath,
+    [string]$Selection,
+    [Parameter(Mandatory=$False)][string]$DisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$Filters,
+    [Parameter(Mandatory=$False)][array]$Replace,
+    [Parameter(Mandatory=$False)][array]$NonSparseParameters
+) {
+    $yamlConfig = Get-Content $matrixFilePath -Raw
+
+    $prettyMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" `
+        -ConfigPath $matrixFilePath `
+        -Selection $Selection `
+        -DisplayNameFilter $DisplayNameFilter `
+        -Filters $Filters `
+        -Replace $Replace `
+        -NonSparseParameters $NonSparseParameters
+    Write-Host $prettyMatrix
+    $prettyMatrix = $prettyMatrix | ConvertFrom-Json
+
+    $scenariosMatrix = @()
+    foreach($permutation in $prettyMatrix.psobject.properties) {
+        $entry = @{}
+        $entry.Name = $permutation.Name -replace '_', '-'
+        $entry.Scenario = $entry.Name
+        $entry.Remove("Name")
+        foreach ($param in $permutation.value.psobject.properties) {
+            $entry.add($param.Name, $param.value)
+        }
+        $scenariosMatrix += $entry
+    }
+
+    $valuesYaml = Get-Content -Raw (Join-Path (Split-Path $matrixFilePath) 'values.yaml')
+    $values = $valuesYaml | ConvertFrom-Yaml -Ordered
+    if (!$values) {$values = @{}}
+
+    if ($values.ContainsKey('Scenarios')) {
+        throw "Please use matrix generation for stress test scenarios."
+    }
+
+    $values.scenarios = $scenariosMatrix
+    $values | ConvertTo-Yaml | Out-File -FilePath (Join-Path $matrixFilePath '../generatedValues.yaml')
+}
+
+function NewStressTestPackageInfo(
+    [hashtable]$chart,
+    [System.IO.FileInfo]$chartFile,
+    [switch]$CI,
+    [object]$namespaceOverride
+) {
+    $namespace = if ($namespaceOverride) {
+        $namespaceOverride
+    } elseif ($CI) {
+        $chart.annotations.namespace
+    } else {
+        GetUsername
+    }
+
+    return [StressTestPackageInfo]@{
+        Namespace = $namespace.ToLower()
+        Directory = $chartFile.DirectoryName
+        ReleaseName = $chart.name
+        Dockerfile = $chart.annotations.dockerfile
+        DockerBuildDir = $chart.annotations.dockerbuilddir
+    }
+}
+
+# Don't call functions when the script is being dot sourced
+if ($MyInvocation.InvocationName -ne ".") {
+    GenerateScenarioMatrix `
+        -matrixFilePath $matrixFilePath `
+        -Selection $Selection `
+        -DisplayNameFilter $DisplayNameFilter `
+        -Filters $Filters `
+        -Replace $Replace `
+        -NonSparseParameters $NonSparseParameters
+}
--- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1
+++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1
@ -79,7 +79,13 @@ function DeployStressTests(
        }
        return $true
    })]
-    [System.IO.FileInfo]$LocalAddonsPath
+    [System.IO.FileInfo]$LocalAddonsPath,
+    [Parameter(Mandatory=$False)][string]$MatrixFileName,
+    [Parameter(Mandatory=$False)][string]$MatrixSelection = "sparse",
+    [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$MatrixFilters,
+    [Parameter(Mandatory=$False)][array]$MatrixReplace,
+    [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters
 ) {
    if ($environment -eq 'pg') {
        if ($clusterGroup -or $subscription) {
@ -115,9 +121,17 @@ function DeployStressTests(

    Run helm repo update
    if ($LASTEXITCODE) { return $LASTEXITCODE }
-
    $deployer = if ($deployId) { $deployId } else { GetUsername }
-    $pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI -namespaceOverride $Namespace
+    $pkgs = @(FindStressPackages `
+                -directory $searchDirectory `
+                -filters $filters `
+                -CI:$CI `
+                -namespaceOverride $Namespace `
+                -MatrixSelection $MatrixSelection `
+                -MatrixFileName $MatrixFileName `
+                -MatrixFilters $MatrixFilters `
+                -MatrixReplace $MatrixReplace `
+                -MatrixNonSparseParameters $MatrixNonSparseParameters)
    Write-Host "" "Found $($pkgs.Length) stress test packages:"
    Write-Host $pkgs.Directory ""
    foreach ($pkg in $pkgs) {
@ -164,59 +178,96 @@ function DeployStressPackage(
        if ($LASTEXITCODE) { return }
    }

-    $imageTag = "${registryName}.azurecr.io"
+    $imageTagBase = "${registryName}.azurecr.io"
    if ($repositoryBase) {
-        $imageTag += "/$repositoryBase"
-    }
-    $imageTag += "/$($pkg.Namespace)/$($pkg.ReleaseName):${deployId}"
-
-    $dockerFilePath = if ($pkg.Dockerfile) {
-        Join-Path $pkg.Directory $pkg.Dockerfile
-    } else {
-        "$($pkg.Directory)/Dockerfile"
-    }
-    $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath)
-
-    if ($pushImages -and (Test-Path $dockerFilePath)) {
-        Write-Host "Building and pushing stress test docker image '$imageTag'"
-        $dockerFile = Get-ChildItem $dockerFilePath
-        $dockerBuildFolder = if ($pkg.DockerBuildDir) {
-            Join-Path $pkg.Directory $pkg.DockerBuildDir
-        } else {
-            $dockerFile.DirectoryName
-        }
-        $dockerBuildFolder = [System.IO.Path]::GetFullPath($dockerBuildFolder).Trim()
-
-        Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder
-        if ($LASTEXITCODE) { return }
-
-        Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue
-        Write-Host "  docker run -it $imageTag" -ForegroundColor DarkBlue
-        Write-Host "  docker run -it $imageTag <shell, e.g. 'bash' 'pwsh' 'sh'>" -ForegroundColor DarkBlue
-        Write-Host "To show installed container images:" -ForegroundColor Blue
-        Write-Host "  docker image ls" -ForegroundColor DarkBlue
-        Write-Host "To show running containers:" -ForegroundColor Blue
-        Write-Host "  docker ps" -ForegroundColor DarkBlue
-
-        Run docker push $imageTag
-        if ($LASTEXITCODE) {
-            if ($login) {
-                Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'"
-            }
-            return
-        }
+        $imageTagBase += "/$repositoryBase"
    }
+    $imageTagBase += "/$($pkg.Namespace)/$($pkg.ReleaseName)"

    Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..."
    kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f -
    if ($LASTEXITCODE) {exit $LASTEXITCODE}

+    $dockerBuildConfigs = @()
+    
+    $genValFile = Join-Path $pkg.Directory "generatedValues.yaml"
+    $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml -Ordered
+    if (Test-Path $genValFile) {
+        $scenarios = $genVal.Scenarios
+        foreach ($scenario in $scenarios) {
+            if ("image" -in $scenario.keys) {
+                $dockerFilePath = Join-Path $pkg.Directory $scenario.image
+            } else {
+                $dockerFilePath = "$($pkg.Directory)/Dockerfile"
+            }
+            $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath).Trim()
+
+            if ("imageBuildDir" -in $scenario.keys) {
+                $dockerBuildDir = Join-Path $pkg.Directory $scenario.imageBuildDir
+            } else {
+                $dockerBuildDir = Split-Path $dockerFilePath
+            }
+            $dockerBuildDir = [System.IO.Path]::GetFullPath($dockerBuildDir).Trim()
+            $dockerBuildConfigs += @{"dockerFilePath"=$dockerFilePath; "dockerBuildDir"=$dockerBuildDir}
+        }
+    }
+    if ($pkg.Dockerfile -or $pkg.DockerBuildDir) {
+        throw "The chart.yaml docker config is depracated, please use the scenarios matrix instead."
+    }
+    
+
+    foreach ($dockerBuildConfig in $dockerBuildConfigs) {
+        $dockerFilePath = $dockerBuildConfig.dockerFilePath
+        $dockerBuildFolder = $dockerBuildConfig.dockerBuildDir
+        if (!(Test-Path $dockerFilePath)) {
+            throw "Invalid dockerfile path, cannot find dockerfile at ${dockerFilePath}"
+        }
+        if (!(Test-Path $dockerBuildFolder)) {
+            throw "Invalid docker build directory, cannot find directory ${dockerBuildFolder}"
+        }
+        $dockerfileName = ($dockerFilePath -split { $_ -in '\', '/' })[-1].ToLower()
+        $imageTag = $imageTagBase + "/${dockerfileName}:${deployId}"
+        if ($pushImages) {
+            Write-Host "Building and pushing stress test docker image '$imageTag'"
+            $dockerFile = Get-ChildItem $dockerFilePath
+
+            Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder
+
+            Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue
+            Write-Host "  docker run -it $imageTag" -ForegroundColor DarkBlue
+            Write-Host "  docker run -it $imageTag <shell, e.g. 'bash' 'pwsh' 'sh'>" -ForegroundColor DarkBlue
+            Write-Host "To show installed container images:" -ForegroundColor Blue
+            Write-Host "  docker image ls" -ForegroundColor DarkBlue
+            Write-Host "To show running containers:" -ForegroundColor Blue
+            Write-Host "  docker ps" -ForegroundColor DarkBlue
+
+            Run docker push $imageTag
+            if ($LASTEXITCODE) {
+                if ($login) {
+                    Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'"
+                }
+            }
+        }
+        $genVal.scenarios = foreach ($scenario in $genVal.scenarios) {
+            $dockerPath = Join-Path $pkg.Directory $scenario.image
+            if ("image" -notin $scenario) {
+                $dockerPath = $dockerFilePath
+            }
+            if ([System.IO.Path]::GetFullPath($dockerPath) -eq $dockerFilePath) {
+                $scenario.imageTag = $imageTag
+            }
+            $scenario
+        }
+
+        $genVal | ConvertTo-Yaml | Out-File -FilePath $genValFile
+    }
+
    Write-Host "Installing or upgrading stress test $($pkg.ReleaseName) from $($pkg.Directory)"
    Run helm upgrade $pkg.ReleaseName $pkg.Directory `
        -n $pkg.Namespace `
        --install `
-        --set image=$imageTag `
-        --set stress-test-addons.env=$environment
+        --set stress-test-addons.env=$environment `
+        --values generatedValues.yaml
    if ($LASTEXITCODE) {
        # Issues like 'UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress'
        # can be the result of cancelled `upgrade` operations (e.g. ctrl-c).