From 438ad5d6f859353385805de20f8d9121b7c9e33a Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Fri, 9 May 2025 21:51:43 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 10579 (#6564) * Support writing .env files from Test Resources If a language repo opts into it *and* if a `test-resources.bicep` file exists and lints clean of writing secrets *and* if the `.env` file is gitignore'd, write a `.env` file next to `test-resources.bicep`. * Resolve PR feedback * Pass -Force for . hidden files on non-Windows --------- Co-authored-by: Heath Stewart --- .../TestResources/New-TestResources.ps1 | 29 +++++--- .../TestResources/New-TestResources.ps1.md | 16 +++-- eng/common/TestResources/README.md | 14 ++-- .../TestResources/Remove-TestResources.ps1 | 1 + .../TestResources/TestResources-Helpers.ps1 | 66 ++++++++++++++++--- eng/common/scripts/common.ps1 | 3 + 6 files changed, 98 insertions(+), 31 deletions(-) diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 7bf1955e1..1e28cdf10 100755 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -122,6 +122,7 @@ param ( $NewTestResourcesRemainingArguments ) +. (Join-Path $PSScriptRoot .. scripts common.ps1) . (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1) . $PSScriptRoot/TestResources-Helpers.ps1 . $PSScriptRoot/SubConfig-Helpers.ps1 @@ -203,11 +204,14 @@ try { $PSBoundParameters['BaseName'] = $BaseName # Try detecting repos that support OutFile and defaulting to it - if (!$CI -and !$PSBoundParameters.ContainsKey('OutFile') -and $IsWindows) { + if (!$CI -and !$PSBoundParameters.ContainsKey('OutFile')) { # TODO: find a better way to detect the language - if (Test-Path "$repositoryRoot/eng/service.proj") { + if ($IsWindows -and $Language -eq 'dotnet') { $OutFile = $true - Log "Detected .NET repository. Defaulting OutFile to true. Test environment settings would be stored into the file so you don't need to set environment variables manually." + Log "Detected .NET repository. Defaulting OutFile to true. Test environment settings will be stored into a file so you don't need to set environment variables manually." + } elseif ($SupportsTestResourcesDotenv) { + $OutFile = $true + Log "Repository supports reading .env files. Defaulting OutFile to true. Test environment settings may be stored in a .env file so they are read by tests automatically." } } @@ -342,10 +346,10 @@ try { if ($context.Account.Type -eq 'User') { # Support corp tenant and TME tenant user id lookups $user = Get-AzADUser -Mail $context.Account.Id - if ($user -eq $null -or !$user.Id) { + if ($null -eq $user -or !$user.Id) { $user = Get-AzADUser -UserPrincipalName $context.Account.Id } - if ($user -eq $null -or !$user.Id) { + if ($null -eq $user -or !$user.Id) { throw "Failed to find entra object ID for the current user" } $ProvisionerApplicationOid = $user.Id @@ -419,10 +423,10 @@ try { # Support corp tenant and TME tenant user id lookups $userAccount = (Get-AzADUser -Mail (Get-AzContext).Account.Id) - if ($userAccount -eq $null -or !$userAccount.Id) { + if ($null -eq $userAccount -or !$userAccount.Id) { $userAccount = (Get-AzADUser -UserPrincipalName (Get-AzContext).Account) } - if ($userAccount -eq $null -or !$userAccount.Id) { + if ($null -eq $userAccount -or !$userAccount.Id) { throw "Failed to find entra object ID for the current user" } $TestApplicationOid = $userAccount.Id @@ -860,14 +864,19 @@ Force creation of resources instead of being prompted. .PARAMETER OutFile Save test environment settings into a .env file next to test resources template. -The contents of the file are protected via the .NET Data Protection API (DPAPI). -This is supported only on Windows. The environment file is scoped to the current -service directory. +On Windows in the Azure/azure-sdk-for-net repository, +the contents of the file are protected via the .NET Data Protection API (DPAPI). +The environment file is scoped to the current service directory. The environment file will be named for the test resources template that it was generated for. For ARM templates, it will be test-resources.json.env. For Bicep templates, test-resources.bicep.env. +If `$SupportsTestResourcesDotenv=$true` in language repos' `LanguageSettings.ps1`, +and if `.env` files are gitignore'd, and if a service directory's `test-resources.bicep` +file does not expose secrets based on `bicep lint`, a `.env` file is written next to +`test-resources.bicep` that can be loaded by a test harness to be used for recording tests. + .PARAMETER SuppressVsoCommands By default, the -CI parameter will print out secrets to logs with Azure Pipelines log commands that cause them to be redacted. For CI environments that don't support this (like diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md index f44feb1ab..6b479f23f 100644 --- a/eng/common/TestResources/New-TestResources.ps1.md +++ b/eng/common/TestResources/New-TestResources.ps1.md @@ -588,17 +588,19 @@ Accept wildcard characters: False ### -OutFile Save test environment settings into a .env file next to test resources template. -The contents of the file are protected via the .NET Data Protection API (DPAPI). -This is supported only on Windows. -The environment file is scoped to the current -service directory. +On Windows in the Azure/azure-sdk-for-net repository, +the contents of the file are protected via the .NET Data Protection API (DPAPI). +The environment file is scoped to the current service directory. The environment file will be named for the test resources template that it was -generated for. -For ARM templates, it will be test-resources.json.env. -For +generated for. For ARM templates, it will be test-resources.json.env. For Bicep templates, test-resources.bicep.env. +If `$SupportsTestResourcesDotenv=$true` in language repos' `LanguageSettings.ps1`, +and if `.env` files are gitignore'd, and if a service directory's `test-resources.bicep` +file does not expose secrets based on `bicep lint`, a `.env` file is written next to +`test-resources.bicep` that can be loaded by a test harness to be used for recording tests. + ```yaml Type: SwitchParameter Parameter Sets: (All) diff --git a/eng/common/TestResources/README.md b/eng/common/TestResources/README.md index b63307e24..8b801963a 100644 --- a/eng/common/TestResources/README.md +++ b/eng/common/TestResources/README.md @@ -1,7 +1,7 @@ # Live Test Resource Management Running and recording live tests often requires first creating some resources -in Azure. Service directories that include a `test-resources.json` or `test-resources.bicep` +in Azure. Service directories that include a `test-resources.json` or `test-resources.bicep` file require running [New-TestResources.ps1][] to create these resources and output environment variables you must set. @@ -19,8 +19,8 @@ scenarios as well as on hosted agents for continuous integration testing. ## On the Desktop To set up your Azure account to run live tests, you'll need to log into Azure, -and create the resources defined in your `test-resources.json` or `test-resources.bicep` -template as shown in the following example using Azure Key Vault. The script will create +and create the resources defined in your `test-resources.json` or `test-resources.bicep` +template as shown in the following example using Azure Key Vault. The script will create a service principal automatically, or you may create a service principal that can be reused subsequently. @@ -34,12 +34,16 @@ Connect-AzAccount -Subscription 'YOUR SUBSCRIPTION ID' eng\common\TestResources\New-TestResources.ps1 keyvault ``` -The `OutFile` switch will be set by default if you are running this for a .NET project on Windows. -This will save test environment settings into a `test-resources.json.env` file next to `test-resources.json` +The `OutFile` switch will be set by default if you are running this for a .NET project on Windows. +This will save test environment settings into a `test-resources.json.env` file next to `test-resources.json` or a `test-resources.bicep.env` file next to `test-resources.bicep`. The file is protected via DPAPI. The environment file would be scoped to the current repository directory and avoids the need to set environment variables or restart your IDE to recognize them. +It will also be set by default for other repositories and on other platforms if your `assets.json` +file contains `"Dotenv": true`. It must be in your `.gitignore` file; +otherwise, an error is returned and no file is generated. + Along with some log messages, this will output environment variables based on your current shell like in the following example: diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index 12411c4ee..4479d9783 100755 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -235,6 +235,7 @@ if ($ServiceDirectory) { # Make sure environment files from New-TestResources -OutFile are removed. Get-ChildItem -Path $root -Filter "$ResourceType-resources.json.env" -Recurse | Remove-Item -Force:$Force + Get-ChildItem -Path $root -Filter ".env" -Recurse -Force | Remove-Item -Force } $verifyDeleteScript = { diff --git a/eng/common/TestResources/TestResources-Helpers.ps1 b/eng/common/TestResources/TestResources-Helpers.ps1 index 400cafcef..5968cac5d 100644 --- a/eng/common/TestResources/TestResources-Helpers.ps1 +++ b/eng/common/TestResources/TestResources-Helpers.ps1 @@ -149,6 +149,33 @@ function BuildBicepFile([System.IO.FileSystemInfo] $file) { return $templateFilePath } +function LintBicepFile([string] $path) { + if (!(Get-Command bicep -ErrorAction Ignore)) { + Write-Error "A bicep file was found at '$path' but the Azure Bicep CLI is not installed. See https://aka.ms/bicep-install" + throw + } + + # Work around lack of config file override: https://github.com/Azure/bicep/issues/5013 + $output = bicep lint "$path" 2>&1 + if ($LASTEXITCODE) { + Write-Error "Failed linting bicep file '$path'" + throw + } + + $clean = $true + foreach ($line in $output) { + $line = $line.ToString() + + # See https://learn.microsoft.com/azure/azure-resource-manager/bicep/bicep-config-linter for lints. + if ($line.Contains('outputs-should-not-contain-secrets')) { + $clean = $false + } + Write-Warning $line + } + + $clean +} + function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) { $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName # Add default values @@ -203,19 +230,40 @@ function SetDeploymentOutputs( $deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables if ($OutFile) { - if (!$IsWindows) { - Write-Host 'File option is supported only on Windows' + if ($IsWindows -and $Language -eq 'dotnet') { + $outputFile = "$($templateFile.originalFilePath).env" + + $environmentText = $deploymentOutputs | ConvertTo-Json; + $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) + $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + + Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force + + Write-Host "Test environment settings`n$environmentText`nstored into encrypted $outputFile" } + elseif ($templateFile.originalFilePath -and $templateFile.originalFilePath.EndsWith(".bicep")) { + $bicepTemplateFile = $templateFile.originalFilePath - $outputFile = "$($templateFile.originalFilePath).env" + # Make sure the file would not write secrets to .env file. + if (!(LintBicepFile $bicepTemplateFile)) { + Write-Error "$bicepTemplateFile may write secrets. No file written." + } + $outputFile = $bicepTemplateFile | Split-Path | Join-Path -ChildPath '.env' - $environmentText = $deploymentOutputs | ConvertTo-Json; - $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) - $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + # Make sure the file would be ignored. + git check-ignore -- "$outputFile" > $null + if ($?) { + $environmentText = foreach ($kv in $deploymentOutputs.GetEnumerator()) { + "$($kv.Key)=`"$($kv.Value)`"" + } - Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force - - Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" + Set-Content $outputFile -Value $environmentText -Force + Write-Host "Test environment settings`n$environmentText`nstored in $outputFile" + } + else { + Write-Error "$outputFile is not ignored by .gitignore. No file written." + } + } } else { if (!$CI) { diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index a8c38d0a0..f8fda7e06 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -24,6 +24,9 @@ $PackageRepository = "Unknown" $packagePattern = "Unknown" $MetadataUri = "Unknown" +# Whether the language repo supports automatically loading .env file generated from TestResources scripts. +$SupportsTestResourcesDotenv = $false + # Import common language settings $EngScriptsLanguageSettings = Join-path $EngScriptsDir "Language-Settings.ps1" if (Test-Path $EngScriptsLanguageSettings) {