Sync eng/common directory with azure-sdk-tools for PR 8422 (#5712)

* Merge subscription configuration file values with the supplied subscription config

* Bring forward working parts

* Collapse into a single task

* SubConfig-Helpers.ps1 changes

* Update remove-test-resources.yml

---------

Co-authored-by: Daniel Jurek <djurek@microsoft.com>

eng/common/TestResources/SubConfig-Helpers.ps1

@@ -103,7 +103,7 @@ function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$va
 
 function SetSubscriptionConfiguration([object]$subscriptionConfiguration)
 {
-    foreach($pair in $subscriptionConfiguration.GetEnumerator()) {
+    foreach ($pair in $subscriptionConfiguration.GetEnumerator()) {
         if ($pair.Value -is [Hashtable]) {
             foreach($nestedPair in $pair.Value.GetEnumerator()) {
                 # Mark values as secret so we don't print json blobs containing secrets in the logs.
@@ -126,36 +126,32 @@ function SetSubscriptionConfiguration([object]$subscriptionConfiguration)
         }
     }
 
-    Write-Host ($subscriptionConfiguration | ConvertTo-Json)
-    $serialized = $subscriptionConfiguration | ConvertTo-Json -Compress
-    Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
+    return $subscriptionConfiguration
 }
 
 function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration)
 {
-        foreach ($pair in $subscriptionConfiguration.GetEnumerator()) {
-            if ($pair.Value -is [Hashtable]) {
-                if (!$subscriptionConfigurationBase.ContainsKey($pair.Name)) {
-                    $subscriptionConfigurationBase[$pair.Name] = @{}
-                }
-                foreach($nestedPair in $pair.Value.GetEnumerator()) {
-                    # Mark values as secret so we don't print json blobs containing secrets in the logs.
-                    # Prepend underscore to the variable name, so we can still access the variable names via environment
-                    # variables if they get set subsequently.
-                    if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) {
-                        Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)"
-                    }
-                    $subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value
-                }
-            } else {
-                if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) {
-                    Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)"
-                }
-                $subscriptionConfigurationBase[$pair.Name] = $pair.Value
+    foreach ($pair in $subscriptionConfiguration.GetEnumerator()) {
+        if ($pair.Value -is [Hashtable]) {
+            if (!$subscriptionConfigurationBase.ContainsKey($pair.Name)) {
+                $subscriptionConfigurationBase[$pair.Name] = @{}
             }
+            foreach($nestedPair in $pair.Value.GetEnumerator()) {
+                # Mark values as secret so we don't print json blobs containing secrets in the logs.
+                # Prepend underscore to the variable name, so we can still access the variable names via environment
+                # variables if they get set subsequently.
+                if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) {
+                    Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)"
+                }
+                $subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value
+            }
+        } else {
+            if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) {
+                Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)"
+            }
+            $subscriptionConfigurationBase[$pair.Name] = $pair.Value
         }
+    }
 
-        $serialized = $subscriptionConfigurationBase | ConvertTo-Json -Compress
-        Write-Host ($subscriptionConfigurationBase | ConvertTo-Json)
-        Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
+    return $subscriptionConfigurationBase
 }

eng/common/TestResources/build-test-resource-config.yml

@@ -9,38 +9,52 @@ parameters:
   - name: EnvVars
     type: object
     default: null
+  - name: SubscriptionConfigurationFilePaths
+    type: object
+    default: null
 
 steps:
-  - ${{ if parameters.SubscriptionConfiguration }}:
-    - pwsh: |
-        $config = @'
-          ${{ parameters.SubscriptionConfiguration }}
-        '@ | ConvertFrom-Json -AsHashtable
+  - pwsh: |
+      . ./eng/common/TestResources/SubConfig-Helpers.ps1
 
-        . ./eng/common/TestResources/SubConfig-Helpers.ps1
-        SetSubscriptionConfiguration $config
-      displayName: Initialize SubscriptionConfiguration variable
-      ${{ if parameters.EnvVars }}:
-        env: ${{ parameters.EnvVars }}
+      $finalConfig = @{}
+      $baseSubConfigRaw = @'
+        ${{ parameters.SubscriptionConfiguration }}
+      '@.Trim()
+      if ($baseSubConfigRaw) {
+        $baseSubConfig = $baseSubConfigRaw | ConvertFrom-Json -AsHashtable
 
-  - ${{ if parameters.SubscriptionConfigurations }}:
-    - pwsh: |
-        Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]{}"
-      displayName: Initialize SubscriptionConfiguration variable for merging
-      condition: eq(variables['SubscriptionConfiguration'], '')
+        Write-Host "Setting base sub config"
+        $finalConfig = SetSubscriptionConfiguration $baseSubConfig
+      }
 
-    - ${{ each config in parameters.SubscriptionConfigurations }}:
-      - pwsh: |
-          $configBase = @'
-            $(SubscriptionConfiguration)
-          '@ | ConvertFrom-Json -AsHashtable
-          $config = @'
-            ${{ config }}
-          '@ | ConvertFrom-Json -AsHashtable
+      $subConfigJsonsRaw = @'
+        ${{ convertToJson(parameters.SubscriptionConfigurations) }}
+      '@.Trim() -replace '"{', '{' -replace '}"', '}'
 
-          . ./eng/common/TestResources/SubConfig-Helpers.ps1
-          UpdateSubscriptionConfiguration $configBase $config
+      if ($subConfigJsonsRaw) {
+        $subConfigs = $subConfigJsonsRaw | ConvertFrom-Json -AsHashtable
 
-        displayName: Merge Test Resource Configurations
-        ${{ if parameters.EnvVars }}:
-          env: ${{ parameters.EnvVars }}
+        foreach ($subConfig in $subConfigs) {
+          Write-Host "Merging sub config from list"
+          $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
+        }
+      }
+
+      $subConfigFilesRaw = @'
+        ${{ convertToJson(parameters.SubscriptionConfigurationFilePaths) }}
+      '@.Trim()
+
+      if ($subConfigFilesRaw) {
+        $subConfigFiles = $subConfigFilesRaw | ConvertFrom-Json -AsHashtable
+        foreach ($file in $subConfigFiles) {
+          Write-Host "Merging sub config from file: $file"
+          $subConfig = Get-Content $file | ConvertFrom-Json -AsHashtable
+          $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
+        }
+      }
+
+      Write-Host ($finalConfig | ConvertTo-Json)
+      $serialized = $finalConfig | ConvertTo-Json -Compress
+      Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
+    displayName: Merge subscription configurations

eng/common/TestResources/deploy-test-resources.yml

@@ -8,8 +8,6 @@ parameters:
   ServiceConnection: not-specified
   ResourceType: test
   UseFederatedAuth: false
-  SubscriptionConfigurationFilePath: ''
-
 
 # SubscriptionConfiguration will be splatted into the parameters of the test
 # resources script. It should be JSON in the form:
@@ -56,18 +54,9 @@ steps:
         ScriptType: InlineScript
         Inline: |
           eng/common/scripts/Import-AzModules.ps1
-
-          if ('${{ parameters.SubscriptionConfigurationFilePath }}' -ne '') {
-            $subscriptionConfiguration = `
-              Get-Content '${{ parameters.SubscriptionConfigurationFilePath }}' `
-              | ConvertFrom-Json -AsHashtable;
-          } else {
-            # Multiline string termination ('@) needs to be at the beginning
-            # of the line
-            $subscriptionConfiguration = @'
-              ${{ parameters.SubscriptionConfiguration }}
+          $subscriptionConfiguration = @'
+            ${{ parameters.SubscriptionConfiguration }}
           '@ | ConvertFrom-Json -AsHashtable;
-          }
 
           # The subscriptionConfiguration may have ArmTemplateParameters defined, so
           # pass those in via the ArmTemplateParameters flag, and handle any
@@ -86,7 +75,6 @@ steps:
   - ${{ else }}:
     - pwsh: |
         eng/common/scripts/Import-AzModules.ps1
-
         $subscriptionConfiguration = @'
           ${{ parameters.SubscriptionConfiguration }}
         '@ | ConvertFrom-Json -AsHashtable;

eng/common/TestResources/remove-test-resources.yml

@@ -8,7 +8,6 @@ parameters:
   ResourceType: test
   EnvVars: {}
   UseFederatedAuth: false
-  SubscriptionConfigurationFilePath: ''
 
 # SubscriptionConfiguration will be splat into the parameters of the test
 # resources script. It should be JSON in the form:
@@ -39,20 +38,10 @@ steps:
         Inline: |
           eng/common/scripts/Import-AzModules.ps1
 
-
-          if ('${{ parameters.SubscriptionConfigurationFilePath }}' -ne '') {
-            $subscriptionConfiguration = `
-              Get-Content '${{ parameters.SubscriptionConfigurationFilePath }}' `
-              | ConvertFrom-Json -AsHashtable;
-          } else {
-            # Multiline string termination ("@) needs to be at the beginning
-            # of the line
-            $subscriptionConfiguration = @"
-              ${{ parameters.SubscriptionConfiguration }}
+          $subscriptionConfiguration = @"
+            ${{ parameters.SubscriptionConfiguration }}
           "@ | ConvertFrom-Json -AsHashtable;
 
-          }
-
           eng/common/TestResources/Remove-TestResources.ps1 `
             @subscriptionConfiguration `
             -ResourceType '${{ parameters.ResourceType }}' `

eng/common/TestResources/sub-config/AzurePublicMsft.json

@@ -2,8 +2,10 @@
     "SubscriptionId": "2cd617ea-1866-46b1-90e3-fffb087ebf9b",
     "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47",
     "TestApplicationId": "f850650c-1fcf-4489-b46f-71af2e30d360",
+    "TestApplicationSecret": "",
     "TestApplicationOid": "30511c9d-ba1a-4c7b-b422-5b543da11b3f",
     "ProvisionerApplicationId": "f850650c-1fcf-4489-b46f-71af2e30d360",
+    "ProvisionerApplicationSecret": "",
     "ProvisionerApplicationOid": "30511c9d-ba1a-4c7b-b422-5b543da11b3f",
     "Environment": "AzureCloud",
     "AzureSubscription": "Azure SDK Test Resources"