github/azure-sdk-for-cpp
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Sync eng/common directory with azure-sdk-tools for PR 8760 (#5874)

Browse Source 
* Do not mark values as secret that are from git-hosted sub config files

* Simplify sub config build yaml

---------

Co-authored-by: Ben Broderick Phillips <bebroder@microsoft.com>
This commit is contained in:
Azure SDK Bot 2024-08-06 12:54:24 -07:00 committed by GitHub
parent 4da0180c98
commit 65e2547763
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 72 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
eng/common/TestResources/SubConfig-Helpers.ps1
View File

@ -82,7 +82,8 @@ function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$va
"TenantId",
"TestApplicationId",
"TestApplicationOid",
"ProvisionerApplicationId"
"ProvisionerApplicationId",
"ProvisionerApplicationOid"
)
$serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName
@ -129,7 +130,7 @@ function SetSubscriptionConfiguration([object]$subscriptionConfiguration)
return $subscriptionConfiguration
}
function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration)
function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration, [array]$allowedValues)
{
foreach ($pair in $subscriptionConfiguration.GetEnumerator()) {
if ($pair.Value -is [Hashtable]) {
@ -140,13 +141,13 @@ function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase,
# Mark values as secret so we don't print json blobs containing secrets in the logs.
# Prepend underscore to the variable name, so we can still access the variable names via environment
# variables if they get set subsequently.
if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) {
if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value $allowedValues) {
Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)"
}
$subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value
}
} else {
if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) {
if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value $allowedValues) {
Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)"
}
$subscriptionConfigurationBase[$pair.Name] = $pair.Value
@ -155,3 +156,66 @@ function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase,
return $subscriptionConfigurationBase
}
# Helper function for processing sub config files from a pipeline file list yaml parameter
function UpdateSubscriptionConfigurationWithFiles([object]$baseSubConfig, [string]$fileListJson) {
if (!$fileListJson) {
return $baseSubConfig
}
$finalConfig = $baseSubConfig
$subConfigFiles = $fileListJson | ConvertFrom-Json -AsHashtable
foreach ($file in $subConfigFiles) {
# In some cases, $file could be an empty string. Get-Content will fail
# if $file is an empty string, so skip those cases.
if (!$file) {
continue
}
Write-Host "Merging sub config from file: $file"
$subConfig = Get-Content $file | ConvertFrom-Json -AsHashtable
$allowedValues = @()
# Since the keys are all coming from a file in github, we know every key should not be marked
# as a secret. Set up these exclusions here to make pipeline log debugging easier.
foreach ($pair in $subConfig.GetEnumerator()) {
if ($pair.Value -is [Hashtable]) {
foreach($nestedPair in $pair.Value.GetEnumerator()) {
$allowedValues += $nestedPair.Name
}
} else {
$allowedValues += $pair.Name
}
}
$finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig $allowedValues
}
return $finalConfig
}
# Helper function for processing stringified json sub configs from pipeline parameter data
function BuildAndSetSubscriptionConfig([string]$baseSubConfigJson, [string]$additionalSubConfigsJson, [string]$subConfigFilesJson) {
$finalConfig = @{}
if ($baseSubConfigJson) {
$baseSubConfig = $baseSubConfigJson | ConvertFrom-Json -AsHashtable
Write-Host "Setting base sub config"
$finalConfig = SetSubscriptionConfiguration $baseSubConfig
}
if ($additionalSubConfigsJson) {
$subConfigs = $additionalSubConfigsJson | ConvertFrom-Json -AsHashtable
foreach ($subConfig in $subConfigs) {
Write-Host "Merging sub config from list"
$finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
}
}
Write-Host "Merging sub config from files"
$finalConfig = UpdateSubscriptionConfigurationWithFiles $finalConfig $subConfigFilesJson
Write-Host ($finalConfig | ConvertTo-Json)
$serialized = $finalConfig | ConvertTo-Json -Compress
Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
}

41
eng/common/TestResources/build-test-resource-config.yml
View File

@ -37,50 +37,17 @@ steps:
- pwsh: |
. ./eng/common/TestResources/SubConfig-Helpers.ps1
$finalConfig = @{}
$baseSubConfigRaw = @'
$baseSubConfigJson = @'
${{ parameters.SubscriptionConfiguration }}
'@.Trim()
if ($baseSubConfigRaw) {
$baseSubConfig = $baseSubConfigRaw | ConvertFrom-Json -AsHashtable
Write-Host "Setting base sub config"
$finalConfig = SetSubscriptionConfiguration $baseSubConfig
}
$subConfigJsonsRaw = @'
$additionalSubConfigsJson = @'
${{ convertToJson(parameters.SubscriptionConfigurations) }}
'@.Trim() -replace '"{', '{' -replace '}"', '}'
if ($subConfigJsonsRaw) {
$subConfigs = $subConfigJsonsRaw | ConvertFrom-Json -AsHashtable
foreach ($subConfig in $subConfigs) {
Write-Host "Merging sub config from list"
$finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
}
}
$subConfigFilesRaw = @'
$subConfigFilesJson = @'
${{ convertToJson(parameters.SubscriptionConfigurationFilePaths) }}
'@.Trim()
if ($subConfigFilesRaw) {
$subConfigFiles = $subConfigFilesRaw | ConvertFrom-Json -AsHashtable
foreach ($file in $subConfigFiles) {
# In some cases, $file could be an empty string. Get-Content will fail
# if $file is an empty string, so skip those cases.
if (!$file) {
continue
}
Write-Host "Merging sub config from file: $file"
$subConfig = Get-Content $file | ConvertFrom-Json -AsHashtable
$finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
}
}
Write-Host ($finalConfig | ConvertTo-Json)
$serialized = $finalConfig | ConvertTo-Json -Compress
Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
BuildAndSetSubscriptionConfig $baseSubConfigJson $additionalSubConfigsJson $subConfigFilesJson
displayName: Merge subscription configurations