Sync eng/common directory with azure-sdk-tools for PR 3790 (#4041)

* Added yaml support for job matrix creation

* autogen scenario matrix for stress test

* Temporary Working State

* update to default sparse

* pr comments and some error handling

* custom matrixfilename and ordering of generatedValues.yaml

* common module import

* JobMatrix write host

Co-authored-by: Albert Cheng <albertcheng@microsoft.com>

eng/common/scripts/Helpers/PSModule-Helpers.ps1

@@ -22,9 +22,9 @@ function Update-PSModulePathForCI()
   $modulePaths = $modulePaths.Where({ !$_.StartsWith($hostedAgentModulePath) })
 
   # Add any "az_" paths from the agent which is the lastest set of azure modules
-  $AzModuleCachPath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator
+  $AzModuleCachePath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator
-  if ($AzModuleCachPath -and $env.PSModulePath -notcontains $AzModuleCachPath) {
+  if ($AzModuleCachePath -and $env:PSModulePath -notcontains $AzModuleCachePath) {
-    $modulePaths += $AzModuleCachPath
+    $modulePaths += $AzModuleCachePath
   }
 
   $env:PSModulePath = $modulePaths -join $moduleSeperator

eng/common/scripts/job-matrix/Create-JobMatrix.ps1

@@ -23,7 +23,7 @@ if (!(Test-Path $ConfigPath)) {
     Write-Error "ConfigPath '$ConfigPath' does not exist."
     exit 1
 }
-$config = GetMatrixConfigFromJson (Get-Content $ConfigPath)
+$config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw)
 # Strip empty string filters in order to be able to use azure pipelines yaml join()
 $Filters = $Filters | Where-Object { $_ }
 
@@ -38,4 +38,7 @@ $Filters = $Filters | Where-Object { $_ }
 $serialized = SerializePipelineMatrix $matrix
 
 Write-Output $serialized.pretty
-Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"
+
+if ($null -ne $env:SYSTEM_TEAMPROJECTID) {
+    Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"
+}

eng/common/scripts/job-matrix/job-matrix-functions.ps1

@@ -84,6 +84,7 @@ class MatrixParameter {
     }
 }
 
+. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1)
 $IMPORT_KEYWORD = '$IMPORT'
 
 function GenerateMatrix(
@@ -146,7 +147,7 @@ function ProcessNonSparseParameters(
 
 function FilterMatrixDisplayName([array]$matrix, [string]$filter) {
     return $matrix | Where-Object { $_ } | ForEach-Object {
-        if ($_.Name -match $filter) {
+        if ($_.ContainsKey("Name") -and $_.Name -match $filter) {
             return $_
         }
     }
@@ -168,7 +169,7 @@ function MatchesFilters([hashtable]$entry, [array]$filters) {
         # Default all regex checks to go against empty string when keys are missing.
         # This simplifies the filter syntax/interface to be regex only.
         $value = ""
-        if ($null -ne $entry -and $entry.parameters.Contains($key)) {
+        if ($null -ne $entry -and $entry.ContainsKey("parameters") -and $entry.parameters.Contains($key)) {
             $value = $entry.parameters[$key]
         }
         if ($value -notmatch $regex) {
@@ -190,11 +191,34 @@ function ParseFilter([string]$filter) {
     }
 }
 
-# Importing the JSON as PSCustomObject preserves key ordering,
+function GetMatrixConfigFromFile([String] $config)
-# whereas ConvertFrom-Json -AsHashtable does not
+{
+    [MatrixConfig]$config = try{
+        GetMatrixConfigFromJson $config
+    } catch {
+        GetMatrixConfigFromYaml $config
+    }
+    return $config
+}
+
+function GetMatrixConfigFromYaml([String] $yamlConfig)
+{
+    Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+    # ConvertTo then from json is to make sure the nested values are in PSCustomObject
+    [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json
+    return GetMatrixConfig $config
+}
+
 function GetMatrixConfigFromJson([String]$jsonConfig)
 {
     [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json
+    return GetMatrixConfig $config
+}
+
+# Importing the JSON as PSCustomObject preserves key ordering,
+# whereas ConvertFrom-Json -AsHashtable does not
+function GetMatrixConfig([MatrixConfig]$config)
+{
     $config.matrixParameters = @()
     $config.displayNamesLookup = @{}
     $include = [MatrixParameter[]]@()
@@ -359,7 +383,7 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n
         Write-Error "`$IMPORT path '$importPath' does not exist."
         exit 1
     }
-    $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath)
+    $importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath)
     $importedMatrix = GenerateMatrix `
                         -config $importedMatrixConfig `
                         -selectFromMatrixType $selection `

eng/common/scripts/stress-testing/deploy-stress-tests.ps1

@@ -24,7 +24,15 @@ param(
     [string]$Namespace,
 
     # Override remote stress-test-addons with local on-disk addons for development
-    [System.IO.FileInfo]$LocalAddonsPath
+    [System.IO.FileInfo]$LocalAddonsPath,
+
+    # Matrix generation parameters
+    [Parameter(Mandatory=$False)][string]$MatrixFileName,
+    [Parameter(Mandatory=$False)][string]$MatrixSelection,
+    [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$MatrixFilters,
+    [Parameter(Mandatory=$False)][array]$MatrixReplace,
+    [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters
 )
 
 . $PSScriptRoot/stress-test-deployment-lib.ps1

eng/common/scripts/stress-testing/find-all-stress-packages.ps1

@@ -12,20 +12,42 @@ class StressTestPackageInfo {
     [string]$Deployer
 }
 
+. $PSScriptRoot/../job-matrix/job-matrix-functions.ps1
+. $PSScriptRoot/generate-scenario-matrix.ps1
+
 function FindStressPackages(
     [string]$directory,
     [hashtable]$filters = @{},
     [switch]$CI,
-    [string]$namespaceOverride
+    [string]$namespaceOverride,
+    [string]$MatrixSelection,
+    [Parameter(Mandatory=$False)][string]$MatrixFileName,
+    [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$MatrixFilters,
+    [Parameter(Mandatory=$False)][array]$MatrixReplace,
+    [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters
 ) {
     # Bare minimum filter for stress tests
     $filters['stressTest'] = 'true'
-
     $packages = @()
     $chartFiles = Get-ChildItem -Recurse -Filter 'Chart.yaml' $directory 
+    if (!$MatrixFileName) {
+        $MatrixFileName = '/scenarios-matrix.yaml'
+    }
     foreach ($chartFile in $chartFiles) {
         $chart = ParseChart $chartFile
         if (matchesAnnotations $chart $filters) {
+            $matrixFilePath = (Join-Path $chartFile.Directory.FullName $MatrixFileName)
+            if (Test-Path $matrixFilePath) {
+                GenerateScenarioMatrix `
+                    -matrixFilePath $matrixFilePath `
+                    -Selection $MatrixSelection `
+                    -DisplayNameFilter $MatrixDisplayNameFilter `
+                    -Filters $MatrixFilters `
+                    -Replace $MatrixReplace `
+                    -NonSparseParameters $MatrixNonSparseParameters
+            }
+
             $packages += NewStressTestPackageInfo `
                             -chart $chart `
                             -chartFile $chartFile `
@@ -80,8 +102,8 @@ function NewStressTestPackageInfo(
         Namespace = $namespace.ToLower()
         Directory = $chartFile.DirectoryName
         ReleaseName = $chart.name
-        Dockerfile = $chart.annotations.dockerfile
+        Dockerfile = "dockerfile" -in $chart.annotations.keys ? $chart.annotations.dockerfile : $null
-        DockerBuildDir = $chart.annotations.dockerbuilddir
+        DockerBuildDir = "dockerbuilddir" -in $chart.annotations.keys ? $chart.annotations.dockerbuilddir : $null
     }
 }
 

eng/common/scripts/stress-testing/generate-scenario-matrix.ps1

@@ -0,0 +1,86 @@
+param(
+    [string]$matrixFilePath,
+    [string]$Selection,
+    [Parameter(Mandatory=$False)][string]$DisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$Filters,
+    [Parameter(Mandatory=$False)][array]$Replace,
+    [Parameter(Mandatory=$False)][array]$NonSparseParameters
+)
+
+function GenerateScenarioMatrix(
+    [string]$matrixFilePath,
+    [string]$Selection,
+    [Parameter(Mandatory=$False)][string]$DisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$Filters,
+    [Parameter(Mandatory=$False)][array]$Replace,
+    [Parameter(Mandatory=$False)][array]$NonSparseParameters
+) {
+    $yamlConfig = Get-Content $matrixFilePath -Raw
+
+    $prettyMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" `
+        -ConfigPath $matrixFilePath `
+        -Selection $Selection `
+        -DisplayNameFilter $DisplayNameFilter `
+        -Filters $Filters `
+        -Replace $Replace `
+        -NonSparseParameters $NonSparseParameters
+    Write-Host $prettyMatrix
+    $prettyMatrix = $prettyMatrix | ConvertFrom-Json
+
+    $scenariosMatrix = @()
+    foreach($permutation in $prettyMatrix.psobject.properties) {
+        $entry = @{}
+        $entry.Name = $permutation.Name -replace '_', '-'
+        $entry.Scenario = $entry.Name
+        $entry.Remove("Name")
+        foreach ($param in $permutation.value.psobject.properties) {
+            $entry.add($param.Name, $param.value)
+        }
+        $scenariosMatrix += $entry
+    }
+
+    $valuesYaml = Get-Content -Raw (Join-Path (Split-Path $matrixFilePath) 'values.yaml')
+    $values = $valuesYaml | ConvertFrom-Yaml -Ordered
+    if (!$values) {$values = @{}}
+
+    if ($values.ContainsKey('Scenarios')) {
+        throw "Please use matrix generation for stress test scenarios."
+    }
+
+    $values.scenarios = $scenariosMatrix
+    $values | ConvertTo-Yaml | Out-File -FilePath (Join-Path $matrixFilePath '../generatedValues.yaml')
+}
+
+function NewStressTestPackageInfo(
+    [hashtable]$chart,
+    [System.IO.FileInfo]$chartFile,
+    [switch]$CI,
+    [object]$namespaceOverride
+) {
+    $namespace = if ($namespaceOverride) {
+        $namespaceOverride
+    } elseif ($CI) {
+        $chart.annotations.namespace
+    } else {
+        GetUsername
+    }
+
+    return [StressTestPackageInfo]@{
+        Namespace = $namespace.ToLower()
+        Directory = $chartFile.DirectoryName
+        ReleaseName = $chart.name
+        Dockerfile = $chart.annotations.dockerfile
+        DockerBuildDir = $chart.annotations.dockerbuilddir
+    }
+}
+
+# Don't call functions when the script is being dot sourced
+if ($MyInvocation.InvocationName -ne ".") {
+    GenerateScenarioMatrix `
+        -matrixFilePath $matrixFilePath `
+        -Selection $Selection `
+        -DisplayNameFilter $DisplayNameFilter `
+        -Filters $Filters `
+        -Replace $Replace `
+        -NonSparseParameters $NonSparseParameters
+}

eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1

@@ -79,7 +79,13 @@ function DeployStressTests(
         }
         return $true
     })]
-    [System.IO.FileInfo]$LocalAddonsPath
+    [System.IO.FileInfo]$LocalAddonsPath,
+    [Parameter(Mandatory=$False)][string]$MatrixFileName,
+    [Parameter(Mandatory=$False)][string]$MatrixSelection = "sparse",
+    [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter,
+    [Parameter(Mandatory=$False)][array]$MatrixFilters,
+    [Parameter(Mandatory=$False)][array]$MatrixReplace,
+    [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters
 ) {
     if ($environment -eq 'pg') {
         if ($clusterGroup -or $subscription) {
@@ -115,9 +121,17 @@ function DeployStressTests(
 
     Run helm repo update
     if ($LASTEXITCODE) { return $LASTEXITCODE }
-
     $deployer = if ($deployId) { $deployId } else { GetUsername }
-    $pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI -namespaceOverride $Namespace
+    $pkgs = @(FindStressPackages `
+                -directory $searchDirectory `
+                -filters $filters `
+                -CI:$CI `
+                -namespaceOverride $Namespace `
+                -MatrixSelection $MatrixSelection `
+                -MatrixFileName $MatrixFileName `
+                -MatrixFilters $MatrixFilters `
+                -MatrixReplace $MatrixReplace `
+                -MatrixNonSparseParameters $MatrixNonSparseParameters)
     Write-Host "" "Found $($pkgs.Length) stress test packages:"
     Write-Host $pkgs.Directory ""
     foreach ($pkg in $pkgs) {
@@ -164,59 +178,96 @@ function DeployStressPackage(
         if ($LASTEXITCODE) { return }
     }
 
-    $imageTag = "${registryName}.azurecr.io"
+    $imageTagBase = "${registryName}.azurecr.io"
     if ($repositoryBase) {
-        $imageTag += "/$repositoryBase"
+        $imageTagBase += "/$repositoryBase"
-    }
-    $imageTag += "/$($pkg.Namespace)/$($pkg.ReleaseName):${deployId}"
-
-    $dockerFilePath = if ($pkg.Dockerfile) {
-        Join-Path $pkg.Directory $pkg.Dockerfile
-    } else {
-        "$($pkg.Directory)/Dockerfile"
-    }
-    $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath)
-
-    if ($pushImages -and (Test-Path $dockerFilePath)) {
-        Write-Host "Building and pushing stress test docker image '$imageTag'"
-        $dockerFile = Get-ChildItem $dockerFilePath
-        $dockerBuildFolder = if ($pkg.DockerBuildDir) {
-            Join-Path $pkg.Directory $pkg.DockerBuildDir
-        } else {
-            $dockerFile.DirectoryName
-        }
-        $dockerBuildFolder = [System.IO.Path]::GetFullPath($dockerBuildFolder).Trim()
-
-        Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder
-        if ($LASTEXITCODE) { return }
-
-        Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue
-        Write-Host "  docker run -it $imageTag" -ForegroundColor DarkBlue
-        Write-Host "  docker run -it $imageTag <shell, e.g. 'bash' 'pwsh' 'sh'>" -ForegroundColor DarkBlue
-        Write-Host "To show installed container images:" -ForegroundColor Blue
-        Write-Host "  docker image ls" -ForegroundColor DarkBlue
-        Write-Host "To show running containers:" -ForegroundColor Blue
-        Write-Host "  docker ps" -ForegroundColor DarkBlue
-
-        Run docker push $imageTag
-        if ($LASTEXITCODE) {
-            if ($login) {
-                Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'"
-            }
-            return
-        }
     }
+    $imageTagBase += "/$($pkg.Namespace)/$($pkg.ReleaseName)"
 
     Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..."
     kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f -
     if ($LASTEXITCODE) {exit $LASTEXITCODE}
 
+    $dockerBuildConfigs = @()
+    
+    $genValFile = Join-Path $pkg.Directory "generatedValues.yaml"
+    $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml -Ordered
+    if (Test-Path $genValFile) {
+        $scenarios = $genVal.Scenarios
+        foreach ($scenario in $scenarios) {
+            if ("image" -in $scenario.keys) {
+                $dockerFilePath = Join-Path $pkg.Directory $scenario.image
+            } else {
+                $dockerFilePath = "$($pkg.Directory)/Dockerfile"
+            }
+            $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath).Trim()
+
+            if ("imageBuildDir" -in $scenario.keys) {
+                $dockerBuildDir = Join-Path $pkg.Directory $scenario.imageBuildDir
+            } else {
+                $dockerBuildDir = Split-Path $dockerFilePath
+            }
+            $dockerBuildDir = [System.IO.Path]::GetFullPath($dockerBuildDir).Trim()
+            $dockerBuildConfigs += @{"dockerFilePath"=$dockerFilePath; "dockerBuildDir"=$dockerBuildDir}
+        }
+    }
+    if ($pkg.Dockerfile -or $pkg.DockerBuildDir) {
+        throw "The chart.yaml docker config is depracated, please use the scenarios matrix instead."
+    }
+    
+
+    foreach ($dockerBuildConfig in $dockerBuildConfigs) {
+        $dockerFilePath = $dockerBuildConfig.dockerFilePath
+        $dockerBuildFolder = $dockerBuildConfig.dockerBuildDir
+        if (!(Test-Path $dockerFilePath)) {
+            throw "Invalid dockerfile path, cannot find dockerfile at ${dockerFilePath}"
+        }
+        if (!(Test-Path $dockerBuildFolder)) {
+            throw "Invalid docker build directory, cannot find directory ${dockerBuildFolder}"
+        }
+        $dockerfileName = ($dockerFilePath -split { $_ -in '\', '/' })[-1].ToLower()
+        $imageTag = $imageTagBase + "/${dockerfileName}:${deployId}"
+        if ($pushImages) {
+            Write-Host "Building and pushing stress test docker image '$imageTag'"
+            $dockerFile = Get-ChildItem $dockerFilePath
+
+            Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder
+
+            Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue
+            Write-Host "  docker run -it $imageTag" -ForegroundColor DarkBlue
+            Write-Host "  docker run -it $imageTag <shell, e.g. 'bash' 'pwsh' 'sh'>" -ForegroundColor DarkBlue
+            Write-Host "To show installed container images:" -ForegroundColor Blue
+            Write-Host "  docker image ls" -ForegroundColor DarkBlue
+            Write-Host "To show running containers:" -ForegroundColor Blue
+            Write-Host "  docker ps" -ForegroundColor DarkBlue
+
+            Run docker push $imageTag
+            if ($LASTEXITCODE) {
+                if ($login) {
+                    Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'"
+                }
+            }
+        }
+        $genVal.scenarios = foreach ($scenario in $genVal.scenarios) {
+            $dockerPath = Join-Path $pkg.Directory $scenario.image
+            if ("image" -notin $scenario) {
+                $dockerPath = $dockerFilePath
+            }
+            if ([System.IO.Path]::GetFullPath($dockerPath) -eq $dockerFilePath) {
+                $scenario.imageTag = $imageTag
+            }
+            $scenario
+        }
+
+        $genVal | ConvertTo-Yaml | Out-File -FilePath $genValFile
+    }
+
     Write-Host "Installing or upgrading stress test $($pkg.ReleaseName) from $($pkg.Directory)"
     Run helm upgrade $pkg.ReleaseName $pkg.Directory `
         -n $pkg.Namespace `
         --install `
-        --set image=$imageTag `
+        --set stress-test-addons.env=$environment `
-        --set stress-test-addons.env=$environment
+        --values generatedValues.yaml
     if ($LASTEXITCODE) {
         # Issues like 'UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress'
         # can be the result of cancelled `upgrade` operations (e.g. ctrl-c).