github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Sync eng/common directory with azure-sdk-tools for PR 8558 (#5764)

Browse Source 
* Support storage network access and worm removal in remove test resources script

* Move storage network access script to common resource helpers file

* Improve storage container deletion resilience

* Plumb through pool variable to live test cleanup template

* Add sleep for network rule application

---------

Co-authored-by: Ben Broderick Phillips <bebroder@microsoft.com>
This commit is contained in:
Azure SDK Bot 2024-07-08 17:56:18 -04:00 committed by GitHub
parent de7197daae
commit bc547c4f4c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 501 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

294
eng/common/TestResources/New-TestResources.ps1
View File

@ -117,6 +117,8 @@ param (
$NewTestResourcesRemainingArguments
)
. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1)
. $PSScriptRoot/TestResources-Helpers.ps1
. $PSScriptRoot/SubConfig-Helpers.ps1
if (!$ServicePrincipalAuth) {
@ -131,272 +133,6 @@ if (!$PSBoundParameters.ContainsKey('ErrorAction')) {
$ErrorActionPreference = 'Stop'
}
function Log($Message)
{
Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
}
# vso commands are specially formatted log lines that are parsed by Azure Pipelines
# to perform additional actions, most commonly marking values as secrets.
# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands
function LogVsoCommand([string]$message)
{
if (!$CI -or $SuppressVsoCommands) {
return
}
Write-Host $message
}
function Retry([scriptblock] $Action, [int] $Attempts = 5)
{
$attempt = 0
$sleep = 5
while ($attempt -lt $Attempts) {
try {
$attempt++
return $Action.Invoke()
} catch {
if ($attempt -lt $Attempts) {
$sleep *= 2
Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..."
Start-Sleep -Seconds $sleep
} else {
throw
}
}
}
}
# NewServicePrincipalWrapper creates an object from an AAD graph or Microsoft Graph service principal object type.
# This is necessary to work around breaking changes introduced in Az version 7.0.0:
# https://azure.microsoft.com/en-us/updates/update-your-apps-to-use-microsoft-graph-before-30-june-2022/
function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGroup, [string]$displayName)
{
if ((Get-Module Az.Resources).Version -eq "5.3.0") {
# https://github.com/Azure/azure-powershell/issues/17040
# New-AzAdServicePrincipal calls will fail with:
# "You cannot call a method on a null-valued expression."
Write-Warning "Az.Resources version 5.3.0 is not supported. Please update to >= 5.3.1"
Write-Warning "Update-Module Az.Resources -RequiredVersion 5.3.1"
exit 1
}
try {
$servicePrincipal = Retry {
New-AzADServicePrincipal -Role "Owner" -Scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" -DisplayName $displayName
}
} catch {
# The underlying error "The directory object quota limit for the Principal has been exceeded" gets overwritten by the module trying
# to call New-AzADApplication with a null object instead of stopping execution, which makes this case hard to diagnose because it prints the following:
# "Cannot bind argument to parameter 'ObjectId' because it is an empty string."
# Provide a more helpful diagnostic prompt to the user if appropriate:
$totalApps = (Get-AzADApplication -OwnedApplication).Length
$msg = "App Registrations owned by you total $totalApps and may exceed the max quota (likely around 135)." + `
"`nTry removing some at https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps" + `
" or by running the following command to remove apps created by this script:" + `
"`n Get-AzADApplication -DisplayNameStartsWith '$baseName' | Remove-AzADApplication" + `
"`nNOTE: You may need to wait for the quota number to be updated after removing unused applications."
Write-Warning $msg
throw
}
$spPassword = ""
$appId = ""
if (Get-Member -Name "Secret" -InputObject $servicePrincipal -MemberType property) {
Write-Verbose "Using legacy PSADServicePrincipal object type from AAD graph API"
# Secret property exists on PSADServicePrincipal type from AAD graph in Az # module versions < 7.0.0
$spPassword = $servicePrincipal.Secret
$appId = $servicePrincipal.ApplicationId
} else {
if ((Get-Module Az.Resources).Version -eq "5.1.0") {
Write-Verbose "Creating password and credential for service principal via MS Graph API"
Write-Warning "Please update Az.Resources to >= 5.2.0 by running 'Update-Module Az'"
# Microsoft graph objects (Az.Resources version == 5.1.0) do not provision a secret on creation so it must be added separately.
# Submitting a password credential object without specifying a password will result in one being generated on the server side.
$password = New-Object -TypeName "Microsoft.Azure.PowerShell.Cmdlets.Resources.MSGraph.Models.ApiV10.MicrosoftGraphPasswordCredential"
$password.DisplayName = "Password for $displayName"
$credential = Retry { New-AzADSpCredential -PasswordCredentials $password -ServicePrincipalObject $servicePrincipal -ErrorAction 'Stop' }
$spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force
$appId = $servicePrincipal.AppId
} else {
Write-Verbose "Creating service principal credential via MS Graph API"
# In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the
# parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter.
$credential = Retry { $servicePrincipal | New-AzADSpCredential -ErrorAction 'Stop' }
$spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force
$appId = $servicePrincipal.AppId
}
}
return @{
AppId = $appId
ApplicationId = $appId
# This is the ObjectId/OID but most return objects use .Id so keep it consistent to prevent confusion
Id = $servicePrincipal.Id
DisplayName = $servicePrincipal.DisplayName
Secret = $spPassword
}
}
function LoadCloudConfig([string] $env)
{
$configPath = "$PSScriptRoot/clouds/$env.json"
if (!(Test-Path $configPath)) {
Write-Warning "Could not find cloud configuration for environment '$env'"
return @{}
}
$config = Get-Content $configPath | ConvertFrom-Json -AsHashtable
return $config
}
function MergeHashes([hashtable] $source, [psvariable] $dest)
{
foreach ($key in $source.Keys) {
if ($dest.Value.Contains($key) -and $dest.Value[$key] -ne $source[$key]) {
Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " +
"to new value '$($source[$key])'")
}
$dest.Value[$key] = $source[$key]
}
}
function BuildBicepFile([System.IO.FileSystemInfo] $file)
{
if (!(Get-Command bicep -ErrorAction Ignore)) {
Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See aka.ms/bicep-install"
throw
}
$tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath()
$templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json"
# Az can deploy bicep files natively, but by compiling here it becomes easier to parse the
# outputted json for mismatched parameter declarations.
bicep build $file.FullName --outfile $templateFilePath
if ($LASTEXITCODE) {
Write-Error "Failure building bicep file '$($file.FullName)'"
throw
}
return $templateFilePath
}
function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) {
$serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName
# Add default values
$deploymentOutputs = [Ordered]@{
"${serviceDirectoryPrefix}SUBSCRIPTION_ID" = $azContext.Subscription.Id;
"${serviceDirectoryPrefix}RESOURCE_GROUP" = $resourceGroup.ResourceGroupName;
"${serviceDirectoryPrefix}LOCATION" = $resourceGroup.Location;
"${serviceDirectoryPrefix}ENVIRONMENT" = $azContext.Environment.Name;
"${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority;
"${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl;
"${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl;
"AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant();
}
if ($ServicePrincipalAuth) {
$deploymentOutputs["${serviceDirectoryPrefix}CLIENT_ID"] = $TestApplicationId;
$deploymentOutputs["${serviceDirectoryPrefix}CLIENT_SECRET"] = $TestApplicationSecret;
$deploymentOutputs["${serviceDirectoryPrefix}TENANT_ID"] = $azContext.Tenant.Id;
}
MergeHashes $environmentVariables $(Get-Variable deploymentOutputs)
foreach ($key in $deployment.Outputs.Keys) {
$variable = $deployment.Outputs[$key]
# Work around bug that makes the first few characters of environment variables be lowercase.
$key = $key.ToUpperInvariant()
if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') {
$deploymentOutputs[$key] = $variable.Value
}
}
# Force capitalization of all keys to avoid Azure Pipelines confusion with
# variable auto-capitalization and OS env var capitalization differences
$capitalized = @{}
foreach ($item in $deploymentOutputs.GetEnumerator()) {
$capitalized[$item.Name.ToUpperInvariant()] = $item.Value
}
return $capitalized
}
function SetDeploymentOutputs(
[string]$serviceName,
[object]$azContext,
[object]$deployment,
[object]$templateFile,
[hashtable]$environmentVariables = @{}
) {
$deploymentEnvironmentVariables = $environmentVariables.Clone()
$deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables
if ($OutFile) {
if (!$IsWindows) {
Write-Host 'File option is supported only on Windows'
}
$outputFile = "$($templateFile.originalFilePath).env"
$environmentText = $deploymentOutputs | ConvertTo-Json;
$bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText)
$protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser)
Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force
Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile"
} else {
if (!$CI) {
# Write an extra new line to isolate the environment variables for easy reading.
Log "Persist the following environment variables based on your detected shell ($shell):`n"
}
# Write overwrite warnings first, since local execution prints a runnable command to export variables
foreach ($key in $deploymentOutputs.Keys) {
if ([Environment]::GetEnvironmentVariable($key)) {
Write-Warning "Deployment outputs will overwrite pre-existing environment variable '$key'"
}
}
# Marking values as secret by allowed keys below is not sufficient, as there may be outputs set in the ARM/bicep
# file that re-mark those values as secret (since all user-provided deployment outputs are treated as secret by default).
# This variable supports a second check on not marking previously allowed keys/values as secret.
$notSecretValues = @()
foreach ($key in $deploymentOutputs.Keys) {
$value = $deploymentOutputs[$key]
$deploymentEnvironmentVariables[$key] = $value
if ($CI) {
if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) {
# Treat all ARM template output variables as secrets since "SecureString" variables do not set values.
# In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below.
LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value"
Write-Host "Setting variable as secret '$key'"
} else {
Write-Host "Setting variable '$key': $value"
$notSecretValues += $value
}
LogVsoCommand "##vso[task.setvariable variable=$key;]$value"
} else {
Write-Host ($shellExportFormat -f $key, $value)
}
}
if ($key) {
# Isolate the environment variables for easy reading.
Write-Host "`n"
$key = $null
}
}
return $deploymentEnvironmentVariables, $deploymentOutputs
}
# Support actions to invoke on exit.
$exitActions = @({
@ -843,31 +579,7 @@ try {
-templateFile $templateFile `
-environmentVariables $EnvironmentVariables
$storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" }
# Add client IP to storage account when running as local user. Pipeline's have their own vnet with access
if ($storageAccounts) {
foreach ($account in $storageAccounts) {
$rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name
if ($rules -and $rules.DefaultAction -eq "Allow") {
Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default"
Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny }
if ($CI -and $env:PoolSubnet) {
Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)"
Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet }
} elseif ($AllowIpRanges) {
Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges"
$ipRanges = $AllowIpRanges | ForEach-Object {
@{ Action = 'allow'; IPAddressOrRange = $_ }
}
Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null }
} elseif (!$CI) {
Write-Host "Enabling access to '$($account.Name)' from client IP"
$clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site
Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null }
}
}
}
}
SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI
$postDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath "$ResourceType-resources-post.ps1"
if (Test-Path $postDeploymentScript) {

19
eng/common/TestResources/Remove-TestResources.ps1
View File

@ -61,6 +61,19 @@ param (
[Parameter()]
[switch] $ServicePrincipalAuth,
# List of CIDR ranges to add to specific resource firewalls, e.g. @(10.100.0.0/16, 10.200.0.0/16)
[Parameter()]
[ValidateCount(0,399)]
[Validatescript({
foreach ($range in $PSItem) {
if ($range -like '*/31' -or $range -like '*/32') {
throw "Firewall IP Ranges cannot contain a /31 or /32 CIDR"
}
}
return $true
})]
[array] $AllowIpRanges = @(),
[Parameter()]
[switch] $Force,
@ -69,6 +82,9 @@ param (
$RemoveTestResourcesRemainingArguments
)
. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1)
. (Join-Path $PSScriptRoot TestResources-Helpers.ps1)
# By default stop for any error.
if (!$PSBoundParameters.ContainsKey('ErrorAction')) {
$ErrorActionPreference = 'Stop'
@ -241,6 +257,9 @@ $verifyDeleteScript = {
# Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty.
$purgeableResources = Get-PurgeableGroupResources $ResourceGroupName
SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -Override -CI:$CI
Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName -CI:$CI
Log "Deleting resource group '$ResourceGroupName'"
if ($Force -and !$purgeableResources) {
Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force -AsJob

267
eng/common/TestResources/TestResources-Helpers.ps1 Normal file
View File

@ -0,0 +1,267 @@
function Log($Message) {
Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
}
# vso commands are specially formatted log lines that are parsed by Azure Pipelines
# to perform additional actions, most commonly marking values as secrets.
# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands
function LogVsoCommand([string]$message) {
if (!$CI -or $SuppressVsoCommands) {
return
}
Write-Host $message
}
function Retry([scriptblock] $Action, [int] $Attempts = 5) {
$attempt = 0
$sleep = 5
while ($attempt -lt $Attempts) {
try {
$attempt++
return $Action.Invoke()
}
catch {
if ($attempt -lt $Attempts) {
$sleep *= 2
Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..."
Start-Sleep -Seconds $sleep
}
else {
throw
}
}
}
}
# NewServicePrincipalWrapper creates an object from an AAD graph or Microsoft Graph service principal object type.
# This is necessary to work around breaking changes introduced in Az version 7.0.0:
# https://azure.microsoft.com/en-us/updates/update-your-apps-to-use-microsoft-graph-before-30-june-2022/
function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGroup, [string]$displayName) {
if ((Get-Module Az.Resources).Version -eq "5.3.0") {
# https://github.com/Azure/azure-powershell/issues/17040
# New-AzAdServicePrincipal calls will fail with:
# "You cannot call a method on a null-valued expression."
Write-Warning "Az.Resources version 5.3.0 is not supported. Please update to >= 5.3.1"
Write-Warning "Update-Module Az.Resources -RequiredVersion 5.3.1"
exit 1
}
try {
$servicePrincipal = Retry {
New-AzADServicePrincipal -Role "Owner" -Scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" -DisplayName $displayName
}
}
catch {
# The underlying error "The directory object quota limit for the Principal has been exceeded" gets overwritten by the module trying
# to call New-AzADApplication with a null object instead of stopping execution, which makes this case hard to diagnose because it prints the following:
# "Cannot bind argument to parameter 'ObjectId' because it is an empty string."
# Provide a more helpful diagnostic prompt to the user if appropriate:
$totalApps = (Get-AzADApplication -OwnedApplication).Length
$msg = "App Registrations owned by you total $totalApps and may exceed the max quota (likely around 135)." + `
"`nTry removing some at https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps" + `
" or by running the following command to remove apps created by this script:" + `
"`n Get-AzADApplication -DisplayNameStartsWith '$baseName' | Remove-AzADApplication" + `
"`nNOTE: You may need to wait for the quota number to be updated after removing unused applications."
Write-Warning $msg
throw
}
$spPassword = ""
$appId = ""
if (Get-Member -Name "Secret" -InputObject $servicePrincipal -MemberType property) {
Write-Verbose "Using legacy PSADServicePrincipal object type from AAD graph API"
# Secret property exists on PSADServicePrincipal type from AAD graph in Az # module versions < 7.0.0
$spPassword = $servicePrincipal.Secret
$appId = $servicePrincipal.ApplicationId
}
else {
if ((Get-Module Az.Resources).Version -eq "5.1.0") {
Write-Verbose "Creating password and credential for service principal via MS Graph API"
Write-Warning "Please update Az.Resources to >= 5.2.0 by running 'Update-Module Az'"
# Microsoft graph objects (Az.Resources version == 5.1.0) do not provision a secret on creation so it must be added separately.
# Submitting a password credential object without specifying a password will result in one being generated on the server side.
$password = New-Object -TypeName "Microsoft.Azure.PowerShell.Cmdlets.Resources.MSGraph.Models.ApiV10.MicrosoftGraphPasswordCredential"
$password.DisplayName = "Password for $displayName"
$credential = Retry { New-AzADSpCredential -PasswordCredentials $password -ServicePrincipalObject $servicePrincipal -ErrorAction 'Stop' }
$spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force
$appId = $servicePrincipal.AppId
}
else {
Write-Verbose "Creating service principal credential via MS Graph API"
# In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the
# parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter.
$credential = Retry { $servicePrincipal | New-AzADSpCredential -ErrorAction 'Stop' }
$spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force
$appId = $servicePrincipal.AppId
}
}
return @{
AppId = $appId
ApplicationId = $appId
# This is the ObjectId/OID but most return objects use .Id so keep it consistent to prevent confusion
Id = $servicePrincipal.Id
DisplayName = $servicePrincipal.DisplayName
Secret = $spPassword
}
}
function LoadCloudConfig([string] $env) {
$configPath = "$PSScriptRoot/clouds/$env.json"
if (!(Test-Path $configPath)) {
Write-Warning "Could not find cloud configuration for environment '$env'"
return @{}
}
$config = Get-Content $configPath | ConvertFrom-Json -AsHashtable
return $config
}
function MergeHashes([hashtable] $source, [psvariable] $dest) {
foreach ($key in $source.Keys) {
if ($dest.Value.Contains($key) -and $dest.Value[$key] -ne $source[$key]) {
Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " +
"to new value '$($source[$key])'")
}
$dest.Value[$key] = $source[$key]
}
}
function BuildBicepFile([System.IO.FileSystemInfo] $file) {
if (!(Get-Command bicep -ErrorAction Ignore)) {
Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See aka.ms/bicep-install"
throw
}
$tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath()
$templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json"
# Az can deploy bicep files natively, but by compiling here it becomes easier to parse the
# outputted json for mismatched parameter declarations.
bicep build $file.FullName --outfile $templateFilePath
if ($LASTEXITCODE) {
Write-Error "Failure building bicep file '$($file.FullName)'"
throw
}
return $templateFilePath
}
function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) {
$serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName
# Add default values
$deploymentOutputs = [Ordered]@{
"${serviceDirectoryPrefix}SUBSCRIPTION_ID" = $azContext.Subscription.Id;
"${serviceDirectoryPrefix}RESOURCE_GROUP" = $resourceGroup.ResourceGroupName;
"${serviceDirectoryPrefix}LOCATION" = $resourceGroup.Location;
"${serviceDirectoryPrefix}ENVIRONMENT" = $azContext.Environment.Name;
"${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority;
"${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl;
"${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl;
"AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant();
}
if ($ServicePrincipalAuth) {
$deploymentOutputs["${serviceDirectoryPrefix}CLIENT_ID"] = $TestApplicationId;
$deploymentOutputs["${serviceDirectoryPrefix}CLIENT_SECRET"] = $TestApplicationSecret;
$deploymentOutputs["${serviceDirectoryPrefix}TENANT_ID"] = $azContext.Tenant.Id;
}
MergeHashes $environmentVariables $(Get-Variable deploymentOutputs)
foreach ($key in $deployment.Outputs.Keys) {
$variable = $deployment.Outputs[$key]
# Work around bug that makes the first few characters of environment variables be lowercase.
$key = $key.ToUpperInvariant()
if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') {
$deploymentOutputs[$key] = $variable.Value
}
}
# Force capitalization of all keys to avoid Azure Pipelines confusion with
# variable auto-capitalization and OS env var capitalization differences
$capitalized = @{}
foreach ($item in $deploymentOutputs.GetEnumerator()) {
$capitalized[$item.Name.ToUpperInvariant()] = $item.Value
}
return $capitalized
}
function SetDeploymentOutputs(
[string]$serviceName,
[object]$azContext,
[object]$deployment,
[object]$templateFile,
[hashtable]$environmentVariables = @{}
) {
$deploymentEnvironmentVariables = $environmentVariables.Clone()
$deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables
if ($OutFile) {
if (!$IsWindows) {
Write-Host 'File option is supported only on Windows'
}
$outputFile = "$($templateFile.originalFilePath).env"
$environmentText = $deploymentOutputs | ConvertTo-Json;
$bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText)
$protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser)
Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force
Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile"
}
else {
if (!$CI) {
# Write an extra new line to isolate the environment variables for easy reading.
Log "Persist the following environment variables based on your detected shell ($shell):`n"
}
# Write overwrite warnings first, since local execution prints a runnable command to export variables
foreach ($key in $deploymentOutputs.Keys) {
if ([Environment]::GetEnvironmentVariable($key)) {
Write-Warning "Deployment outputs will overwrite pre-existing environment variable '$key'"
}
}
# Marking values as secret by allowed keys below is not sufficient, as there may be outputs set in the ARM/bicep
# file that re-mark those values as secret (since all user-provided deployment outputs are treated as secret by default).
# This variable supports a second check on not marking previously allowed keys/values as secret.
$notSecretValues = @()
foreach ($key in $deploymentOutputs.Keys) {
$value = $deploymentOutputs[$key]
$deploymentEnvironmentVariables[$key] = $value
if ($CI) {
if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) {
# Treat all ARM template output variables as secrets since "SecureString" variables do not set values.
# In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below.
LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value"
Write-Host "Setting variable as secret '$key'"
}
else {
Write-Host "Setting variable '$key': $value"
$notSecretValues += $value
}
LogVsoCommand "##vso[task.setvariable variable=$key;]$value"
}
else {
Write-Host ($shellExportFormat -f $key, $value)
}
}
if ($key) {
# Isolate the environment variables for easy reading.
Write-Host "`n"
$key = $null
}
}
return $deploymentEnvironmentVariables, $deploymentOutputs
}

10
eng/common/TestResources/remove-test-resources.yml
View File

@ -29,7 +29,9 @@ steps:
displayName: Remove test resources
condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true'))
continueOnError: true
env: ${{ parameters.EnvVars }}
env:
PoolSubnet: $(PoolSubnet)
${{ insert }}: ${{ parameters.EnvVars }}
inputs:
azureSubscription: ${{ parameters.ServiceConnection }}
azurePowerShellVersion: LatestVersion
@ -46,6 +48,7 @@ steps:
@subscriptionConfiguration `
-ResourceType '${{ parameters.ResourceType }}' `
-ServiceDirectory "${{ parameters.ServiceDirectory }}" `
-AllowIpRanges ('$(azsdk-corp-net-ip-ranges)' -split ',') `
-CI `
-Force `
-Verbose
@ -63,10 +66,13 @@ steps:
-ResourceType '${{ parameters.ResourceType }}' `
-ServiceDirectory "${{ parameters.ServiceDirectory }}" `
-ServicePrincipalAuth `
-AllowIpRanges ('$(azsdk-corp-net-ip-ranges)' -split ',') `
-CI `
-Force `
-Verbose
displayName: Remove test resources
condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true'))
continueOnError: true
env: ${{ parameters.EnvVars }}
env:
PoolSubnet: $(PoolSubnet)
${{ insert }}: ${{ parameters.EnvVars }}

232
eng/common/scripts/Helpers/Resource-Helpers.ps1
View File

@ -4,7 +4,7 @@
function Get-PurgeableGroupResources {
param (
[Parameter(Mandatory=$true, Position=0)]
[Parameter(Mandatory = $true, Position = 0)]
[string] $ResourceGroupName
)
@ -27,8 +27,8 @@ function Get-PurgeableGroupResources {
# Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled.
$deletedKeyVaults = @(Get-AzKeyVault -ResourceGroupName $ResourceGroupName -ErrorAction Ignore | ForEach-Object {
# Enumerating vaults from a resource group does not return all properties we required.
Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } `
# Enumerating vaults from a resource group does not return all properties we required.
Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } `
| Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru `
| Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
})
@ -56,13 +56,13 @@ function Get-PurgeableResources {
$deletedHsms = @()
foreach ($r in $content.value) {
$deletedHsms += [pscustomobject] @{
AzsdkResourceType = 'Managed HSM'
AzsdkName = $r.name
Id = $r.id
Name = $r.name
Location = $r.properties.location
DeletionDate = $r.properties.deletionDate -as [DateTime]
ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime]
AzsdkResourceType = 'Managed HSM'
AzsdkName = $r.name
Id = $r.id
Name = $r.name
Location = $r.properties.location
DeletionDate = $r.properties.deletionDate -as [DateTime]
ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime]
EnablePurgeProtection = $r.properties.purgeProtectionEnabled
}
}
@ -91,7 +91,8 @@ function Get-PurgeableResources {
Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
$purgeableResources += $deletedKeyVaults
}
} catch { }
}
catch { }
return $purgeableResources
}
@ -100,7 +101,7 @@ function Get-PurgeableResources {
# This allows you to pipe a collection and process each item in the collection.
filter Remove-PurgeableResources {
param (
[Parameter(Position=0, ValueFromPipeline=$true)]
[Parameter(Position = 0, ValueFromPipeline = $true)]
[object[]] $Resource,
[Parameter()]
@ -128,7 +129,7 @@ filter Remove-PurgeableResources {
# Use `-AsJob` to start a lightweight, cancellable job and pass to `Wait-PurgeableResoruceJob` for consistent behavior.
Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue -AsJob `
| Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru
| Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru
}
'Managed HSM' {
@ -139,18 +140,19 @@ filter Remove-PurgeableResources {
# Use `GetNewClosure()` on the `-Action` ScriptBlock to make sure variables are captured.
Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2023-02-01" -ErrorAction Ignore -AsJob `
| Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action {
param ( $response )
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
} elseif ($response.Content) {
$content = $response.Content | ConvertFrom-Json
if ($content.error) {
$err = $content.error
Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
}
}
}.GetNewClosure()
| Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action {
param ( $response )
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
}
elseif ($response.Content) {
$content = $response.Content | ConvertFrom-Json
if ($content.error) {
$err = $content.error
Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
}
}
}.GetNewClosure()
}
default {
@ -167,12 +169,12 @@ function Log($Message) {
function Wait-PurgeableResourceJob {
param (
[Parameter(Mandatory=$true, ValueFromPipeline=$true)]
[Parameter(Mandatory = $true, ValueFromPipeline = $true)]
$Job,
# The resource is used for logging and to return if `-PassThru` is specified
# so we can easily see all resources that may be in a bad state when the script has completed.
[Parameter(Mandatory=$true)]
[Parameter(Mandatory = $true)]
$Resource,
# Optional ScriptBlock should define params corresponding to the associated job's `Output` property.
@ -195,7 +197,8 @@ function Wait-PurgeableResourceJob {
if ($Action) {
$null = $Action.Invoke($result)
}
} else {
}
else {
Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'. Cancelling job."
$Job.Cancel()
@ -204,3 +207,176 @@ function Wait-PurgeableResourceJob {
}
}
}
# Helper function for removing storage accounts with WORM that sometimes get leaked from live tests not set up to clean
# up their resource policies
function Remove-WormStorageAccounts() {
[CmdletBinding(SupportsShouldProcess = $True)]
param(
[string]$GroupPrefix,
[switch]$CI
)
$ErrorActionPreference = 'Stop'
# Be a little defensive so we don't delete non-live test groups via naming convention
# DO NOT REMOVE THIS
# We call this script from live test pipelines as well, and a string mismatch/error could blow away
# some static storage accounts we rely on
if (!$groupPrefix -or ($CI -and !$GroupPrefix.StartsWith('rg-'))) {
throw "The -GroupPrefix parameter must not be empty, or must start with 'rg-' in CI contexts"
}
$groups = Get-AzResourceGroup | Where-Object { $_.ResourceGroupName.StartsWith($GroupPrefix) } | Where-Object { $_.ProvisioningState -ne 'Deleting' }
foreach ($group in $groups) {
Write-Host "========================================="
$accounts = Get-AzStorageAccount -ResourceGroupName $group.ResourceGroupName
if ($accounts) {
foreach ($account in $accounts) {
if ($WhatIfPreference) {
Write-Host "What if: Removing $($account.StorageAccountName) in $($account.ResourceGroupName)"
}
else {
Write-Host "Removing $($account.StorageAccountName) in $($account.ResourceGroupName)"
}
$hasContainers = ($account.Kind -ne "FileStorage")
# If it doesn't have containers then we can skip the explicit clean-up of this storage account
if (!$hasContainers) { continue }
$ctx = New-AzStorageContext -StorageAccountName $account.StorageAccountName
$immutableBlobs = $ctx `
| Get-AzStorageContainer `
| Where-Object { $_.BlobContainerProperties.HasImmutableStorageWithVersioning } `
| Get-AzStorageBlob
try {
foreach ($blob in $immutableBlobs) {
Write-Host "Removing legal hold - blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)"
$blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null
}
}
catch {
Write-Warning "User must have 'Storage Blob Data Owner' RBAC permission on subscription or resource group"
Write-Error $_
throw
}
# Sometimes we get a 404 blob not found but can still delete containers,
# and sometimes we must delete the blob if there's a legal hold.
# Try to remove the blob, but keep running regardless.
$succeeded = $false
for ($attempt = 0; $attempt -lt 2; $attempt++) {
if ($succeeded) {
break
}
try {
Write-Host "Removing immutability policies - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)"
$null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy
}
catch {}
try {
$ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force
$succeeded = $true
}
catch {
Write-Warning "Failed to remove blobs - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)"
Write-Warning $_
}
}
try {
# Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs
$ctx | Get-AzStorageContainer | ForEach-Object { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force }
}
catch {
Write-Warning "Container removal failed. Ignoring the error and trying to delete the storage account."
Write-Warning $_
}
Remove-AzStorageAccount -StorageAccountName $account.StorageAccountName -ResourceGroupName $account.ResourceGroupName -Force
}
}
if ($WhatIfPreference) {
Write-Host "What if: Removing resource group $($group.ResourceGroupName)"
}
else {
Remove-AzResourceGroup -ResourceGroupName $group.ResourceGroupName -Force -AsJob
}
}
}
function SetResourceNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI) {
SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI
}
function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI, [switch]$Override) {
$clientIp = $null
$storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" }
# Add client IP to storage account when running as local user. Pipeline's have their own vnet with access
if ($storageAccounts) {
$appliedRule = $false
foreach ($account in $storageAccounts) {
$rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name
if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) {
Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default"
Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny }
if ($CI -and $env:PoolSubnet) {
Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)"
Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet }
$appliedRule = $true
}
elseif ($AllowIpRanges) {
Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges"
$ipRanges = $AllowIpRanges | ForEach-Object {
@{ Action = 'allow'; IPAddressOrRange = $_ }
}
Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null }
$appliedRule = $true
}
elseif (!$CI) {
Write-Host "Enabling access to '$($account.Name)' from client IP"
$clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site
$clientIp = $clientIp.Trim()
$ipRanges = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name
if ($ipRanges) {
foreach ($range in $ipRanges.IpRules) {
if (DoesSubnetOverlap $range.IPAddressOrRange $clientIp) {
return
}
}
}
Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null }
$appliedRule = $true
}
}
}
if ($appliedRule) {
Write-Host "Sleeping for 15 seconds to allow network rules to take effect"
Start-Sleep 15
}
}
}
function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) {
[System.Net.IPAddress]$overlapIpAddress = $overlapIp
$parsed = $ipOrCidr -split '/'
[System.Net.IPAddress]$baseIp = $parsed[0]
if ($parsed.Length -eq 1) {
return $baseIp -eq $overlapIpAddress
}
$subnet = $parsed[1]
$subnetNum = [int]$subnet
$baseMask = [math]::pow(2, 31)
$mask = 0
for ($i = 0; $i -lt $subnetNum; $i++) {
$mask = $mask + $baseMask;
$baseMask = $baseMask / 2
}
return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address)
}