238 lines
8.4 KiB
PowerShell
238 lines
8.4 KiB
PowerShell
$ErrorActionPreference = 'Stop'
|
|
|
|
. $PSScriptRoot/find-all-stress-packages.ps1
|
|
$FailedCommands = New-Object Collections.Generic.List[hashtable]
|
|
|
|
. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1)
|
|
|
|
# Powershell does not (at time of writing) treat exit codes from external binaries
|
|
# as cause for stopping execution, so do this via a wrapper function.
|
|
# See https://github.com/PowerShell/PowerShell-RFC/pull/277
|
|
function Run()
|
|
{
|
|
Write-Host "`n==> $args`n" -ForegroundColor Green
|
|
$command, $arguments = $args
|
|
& $command $arguments
|
|
if ($LASTEXITCODE) {
|
|
Write-Error "Command '$args' failed with code: $LASTEXITCODE" -ErrorAction 'Continue'
|
|
$FailedCommands.Add(@{ command = "$args"; code = $LASTEXITCODE })
|
|
}
|
|
}
|
|
|
|
function RunOrExitOnFailure()
|
|
{
|
|
run @args
|
|
if ($LASTEXITCODE) {
|
|
exit $LASTEXITCODE
|
|
}
|
|
}
|
|
|
|
function Login([string]$subscription, [string]$clusterGroup, [switch]$pushImages)
|
|
{
|
|
Write-Host "Logging in to subscription, cluster and container registry"
|
|
az account show *> $null
|
|
if ($LASTEXITCODE) {
|
|
RunOrExitOnFailure az login --allow-no-subscriptions
|
|
}
|
|
|
|
# Discover cluster name, only one cluster per group is expected
|
|
Write-Host "Listing AKS cluster in $subscription/$clusterGroup"
|
|
$cluster = RunOrExitOnFailure az aks list -g $clusterGroup --subscription $subscription -o json
|
|
$clusterName = ($cluster | ConvertFrom-Json).name
|
|
|
|
$kubeContext = (RunOrExitOnFailure kubectl config view -o json) | ConvertFrom-Json
|
|
$defaultNamespace = $kubeContext.contexts.Where({ $_.name -eq $clusterName }).context.namespace
|
|
|
|
RunOrExitOnFailure az aks get-credentials `
|
|
-n "$clusterName" `
|
|
-g "$clusterGroup" `
|
|
--subscription "$subscription" `
|
|
--overwrite-existing
|
|
|
|
if ($defaultNamespace) {
|
|
RunOrExitOnFailure kubectl config set-context $clusterName --namespace $defaultNamespace
|
|
}
|
|
|
|
if ($pushImages) {
|
|
$registry = RunOrExitOnFailure az acr list -g $clusterGroup --subscription $subscription -o json
|
|
$registryName = ($registry | ConvertFrom-Json).name
|
|
RunOrExitOnFailure az acr login -n $registryName
|
|
}
|
|
}
|
|
|
|
function DeployStressTests(
|
|
[string]$searchDirectory = '.',
|
|
[hashtable]$filters = @{},
|
|
[string]$environment = 'test',
|
|
[string]$repository = '',
|
|
[switch]$pushImages,
|
|
[string]$clusterGroup = '',
|
|
[string]$deployId = 'local',
|
|
[switch]$login,
|
|
[string]$subscription = '',
|
|
[switch]$CI
|
|
) {
|
|
if ($environment -eq 'test') {
|
|
if ($clusterGroup -or $subscription) {
|
|
Write-Warning "Overriding cluster group and subscription with defaults for 'test' environment."
|
|
}
|
|
$clusterGroup = 'rg-stress-cluster-test'
|
|
$subscription = 'Azure SDK Developer Playground'
|
|
} elseif ($environment -eq 'prod') {
|
|
if ($clusterGroup -or $subscription) {
|
|
Write-Warning "Overriding cluster group and subscription with defaults for 'prod' environment."
|
|
}
|
|
$clusterGroup = 'rg-stress-cluster-prod'
|
|
$subscription = 'Azure SDK Test Resources'
|
|
}
|
|
|
|
if (!$repository) {
|
|
$repository = if ($env:USER) { $env:USER } else { "${env:USERNAME}" }
|
|
# Remove spaces, etc. that may be in $namespace
|
|
$repository -replace '\W'
|
|
}
|
|
|
|
if ($login) {
|
|
if (!$clusterGroup -or !$subscription) {
|
|
throw "clusterGroup and subscription parameters must be specified when logging into an environment that is not test or prod."
|
|
}
|
|
Login -subscription $subscription -clusterGroup $clusterGroup -pushImages:$pushImages
|
|
}
|
|
|
|
RunOrExitOnFailure helm repo add stress-test-charts https://stresstestcharts.blob.core.windows.net/helm/
|
|
Run helm repo update
|
|
if ($LASTEXITCODE) { return $LASTEXITCODE }
|
|
|
|
$pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI
|
|
Write-Host "" "Found $($pkgs.Length) stress test packages:"
|
|
Write-Host $pkgs.Directory ""
|
|
foreach ($pkg in $pkgs) {
|
|
Write-Host "Deploying stress test at '$($pkg.Directory)'"
|
|
DeployStressPackage `
|
|
-pkg $pkg `
|
|
-deployId $deployId `
|
|
-environment $environment `
|
|
-repositoryBase $repository `
|
|
-pushImages:$pushImages `
|
|
-login:$login
|
|
}
|
|
|
|
Write-Host "Releases deployed by $deployId"
|
|
Run helm list --all-namespaces -l deployId=$deployId
|
|
|
|
if ($FailedCommands) {
|
|
Write-Warning "The following commands failed:"
|
|
foreach ($cmd in $FailedCommands) {
|
|
Write-Error "'$($cmd.command)' failed with code $($cmd.code)" -ErrorAction 'Continue'
|
|
}
|
|
exit 1
|
|
}
|
|
|
|
Write-Host "`nStress test telemetry links (dashboard, fileshare, etc.): https://aka.ms/azsdk/stress/dashboard"
|
|
}
|
|
|
|
function DeployStressPackage(
|
|
[object]$pkg,
|
|
[string]$deployId,
|
|
[string]$environment,
|
|
[string]$repositoryBase,
|
|
[switch]$pushImages,
|
|
[switch]$login
|
|
) {
|
|
$registry = RunOrExitOnFailure az acr list -g $clusterGroup --subscription $subscription -o json
|
|
$registryName = ($registry | ConvertFrom-Json).name
|
|
|
|
Run helm dependency update $pkg.Directory
|
|
if ($LASTEXITCODE) { return }
|
|
|
|
if (Test-Path "$($pkg.Directory)/stress-test-resources.bicep") {
|
|
Run az bicep build -f "$($pkg.Directory)/stress-test-resources.bicep"
|
|
if ($LASTEXITCODE) { return }
|
|
}
|
|
|
|
$imageTag = "${registryName}.azurecr.io"
|
|
if ($repositoryBase) {
|
|
$imageTag += "/$repositoryBase"
|
|
}
|
|
$imageTag += "/$($pkg.Namespace)/$($pkg.ReleaseName):${deployId}"
|
|
|
|
if ($pushImages) {
|
|
Write-Host "Building and pushing stress test docker image '$imageTag'"
|
|
$dockerFile = Get-ChildItem "$($pkg.Directory)/Dockerfile"
|
|
Run docker build -t $imageTag -f $dockerFile.FullName $dockerFile.DirectoryName
|
|
if ($LASTEXITCODE) { return }
|
|
Run docker push $imageTag
|
|
if ($LASTEXITCODE) {
|
|
if ($login) {
|
|
Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'"
|
|
}
|
|
return
|
|
}
|
|
}
|
|
|
|
Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..."
|
|
kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f -
|
|
if ($LASTEXITCODE) {exit $LASTEXITCODE}
|
|
|
|
Write-Host "Installing or upgrading stress test $($pkg.ReleaseName) from $($pkg.Directory)"
|
|
Run helm upgrade $pkg.ReleaseName $pkg.Directory `
|
|
-n $pkg.Namespace `
|
|
--install `
|
|
--set image=$imageTag `
|
|
--set stress-test-addons.env=$environment
|
|
if ($LASTEXITCODE) {
|
|
# Issues like 'UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress'
|
|
# can be the result of cancelled `upgrade` operations (e.g. ctrl-c).
|
|
# See https://github.com/helm/helm/issues/4558
|
|
Write-Warning "The issue may be fixable by first running 'helm rollback -n $($pkg.Namespace) $($pkg.ReleaseName)'"
|
|
return
|
|
}
|
|
|
|
# Helm 3 stores release information in kubernetes secrets. The only way to add extra labels around
|
|
# specific releases (thereby enabling filtering on `helm list`) is to label the underlying secret resources.
|
|
# There is not currently support for setting these labels via the helm cli.
|
|
$helmReleaseConfig = kubectl get secrets `
|
|
-n $pkg.Namespace `
|
|
-l status=deployed,name=$($pkg.ReleaseName) `
|
|
-o jsonpath='{.items[0].metadata.name}'
|
|
|
|
Run kubectl label secret -n $pkg.Namespace --overwrite $helmReleaseConfig deployId=$deployId
|
|
}
|
|
|
|
function CheckDependencies()
|
|
{
|
|
$deps = @(
|
|
@{
|
|
Command = "docker";
|
|
Help = "Docker must be installed: https://docs.docker.com/get-docker/";
|
|
}
|
|
@{
|
|
Command = "kubectl";
|
|
Help = "kubectl must be installed: https://kubernetes.io/docs/tasks/tools/#kubectl";
|
|
},
|
|
@{
|
|
Command = "helm";
|
|
Help = "helm must be installed: https://helm.sh/docs/intro/install/";
|
|
},
|
|
@{
|
|
Command = "az";
|
|
Help = "Azure CLI must be installed: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli";
|
|
}
|
|
)
|
|
|
|
Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
|
|
|
|
$shouldError = $false
|
|
foreach ($dep in $deps) {
|
|
if (!(Get-Command $dep.Command -ErrorAction SilentlyContinue)) {
|
|
$shouldError = $true
|
|
Write-Error $dep.Help
|
|
}
|
|
}
|
|
|
|
if ($shouldError) {
|
|
exit 1
|
|
}
|
|
|
|
}
|