Sync eng/common directory with azure-sdk-tools for PR 8754 (#5884)

* more compatibility with expanding/contracting packages. add ability for packages to have DependentPackages that must be included in the set of packages that should be built given a changeset


---------

Co-authored-by: Scott Beddall <scbedd@microsoft.com>
Co-authored-by: Scott Beddall <45376673+scbedd@users.noreply.github.com>
Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

eng/common/pipelines/templates/steps/detect-api-changes.yml

@@ -1,6 +1,5 @@
 parameters:
   ArtifactPath: $(Build.ArtifactStagingDirectory)
-  Artifacts: []
   ArtifactName: 'packages'
 
 steps:
@@ -14,7 +13,6 @@ steps:
     inputs:
       filePath: $(Build.SourcesDirectory)/eng/common/scripts/Detect-Api-Changes.ps1
       arguments: >
-        -ArtifactList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object Name)
         -ArtifactPath ${{parameters.ArtifactPath}}
         -CommitSha '$(Build.SourceVersion)'
         -BuildId $(Build.BuildId)

eng/common/pipelines/templates/steps/verify-changelogs.yml

@@ -0,0 +1,16 @@
+parameters:
+- name: PackagePropertiesFolder
+  type: string
+- name: Condition
+  type: string
+  default: succeeded()
+
+steps:
+  - task: Powershell@2
+    inputs:
+      filePath: $(Build.SourcesDirectory)/eng/common/scripts/Verify-ChangeLogs.ps1
+      arguments: >
+        -PackagePropertiesFolder '${{ parameters.PackagePropertiesFolder }}'
+      pwsh: true
+    displayName: Verify ChangeLogEntries
+    condition: ${{ parameters.Condition }}

eng/common/pipelines/templates/steps/verify-readmes.yml

@@ -0,0 +1,43 @@
+parameters:
+- name: PackagePropertiesFolder
+  type: string
+- name: RepoRoot
+  type: string
+  default: $(Build.SourcesDirectory)
+- name: SettingsPath
+  type: string
+  default: '$(Build.SourcesDirectory)/eng/.docsettings.yml'
+- name: DocWardenVersion
+  type: string
+  default: ''
+- name: Condition
+  type: string
+  default: succeeded()
+
+steps:
+- pwsh: |
+    $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackagePropertiesFolder }}" *.json
+
+    $paths = @()
+
+    foreach($propertiesFile in $packageProperties) {
+      $PackageProp = Get-Content -Path $propertiesFile | ConvertFrom-Json
+
+      $paths += (Join-Path "$(Build.SourcesDirectory)" $PackageProp.DirectoryPath)
+    }
+
+    $scanPaths = $paths -join ","
+    Write-Host "##vso[task.setvariable variable=ScanPathArgument;]$scanPaths"
+  displayName: Populate Scan Paths
+
+- task: PowerShell@2
+  displayName: "Verify Readmes"
+  condition: ${{ parameters.Condition }}
+  inputs:
+    filePath: "eng/common/scripts/Verify-Readme.ps1"
+    arguments: >
+      -DocWardenVersion '${{ parameters.DocWardenVersion }}'
+      -ScanPaths '$(ScanPathArgument)'
+      -RepoRoot ${{ parameters.RepoRoot }}
+      -SettingsPath ${{ parameters.SettingsPath }}
+    pwsh: true

eng/common/scripts/Detect-Api-Changes.ps1

@@ -10,8 +10,6 @@ Param (
   [string] $BuildId,
   [Parameter(Mandatory=$True)]
   [string] $CommitSha,
-  [Parameter(Mandatory=$True)]
-  [array] $ArtifactList,
   [string] $APIViewUri,
   [string] $RepoFullName = "",
   [string] $ArtifactName = "packages",
@@ -21,6 +19,8 @@ Param (
 
 . (Join-Path $PSScriptRoot common.ps1)
 
+$configFileDir = Join-Path -Path $ArtifactPath "PackageInfo"
+
 # Submit API review request and return status whether current revision is approved or pending or failed to create review
 function Submit-Request($filePath, $packageName)
 {
@@ -64,7 +64,6 @@ function Submit-Request($filePath, $packageName)
 function Should-Process-Package($pkgPath, $packageName)
 {
     $pkg = Split-Path -Leaf $pkgPath
-    $configFileDir = Join-Path -Path $ArtifactPath "PackageInfo"
     $pkgPropPath = Join-Path -Path $configFileDir "$packageName.json"
     if (!(Test-Path $pkgPropPath))
     {
@@ -103,32 +102,39 @@ if (!($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiR
 }
 
 $responses = @{}
-foreach ($artifact in $ArtifactList)
+
+$packageProperties = Get-ChildItem -Recurse -Force "$configFileDir" `
+  | Where-Object { $_.Extension -eq '.json' }
+
+foreach ($packagePropFile in $packageProperties)
 {
-    Write-Host "Processing $($artifact.name)"
-    $packages = &$FindArtifactForApiReviewFn $ArtifactPath $artifact.name
+    $packageMetadata = Get-Content $packagePropFile | ConvertFrom-Json
+    Write-Host "Processing $($packageMetadata.ArtifactName)"
+
+    $packages = &$FindArtifactForApiReviewFn $ArtifactPath $packageMetadata.ArtifactName
+
     if ($packages)
     {
         $pkgPath = $packages.Values[0]
-        $isRequired = Should-Process-Package -pkgPath $pkgPath -packageName $artifact.name
-        Write-Host "Is API change detect required for $($artifact.name):$($isRequired)"
+        $isRequired = Should-Process-Package -pkgPath $pkgPath -packageName $($packageMetadata.ArtifactName)
+        Write-Host "Is API change detect required for $($packages.ArtifactName):$($isRequired)"
         if ($isRequired -eq $True)
         {
             $filePath = $pkgPath.Replace($ArtifactPath , "").Replace("\", "/")
-            $respCode = Submit-Request -filePath $filePath -packageName $artifact.name
+            $respCode = Submit-Request -filePath $filePath -packageName $($packageMetadata.ArtifactName)
             if ($respCode -ne '200')
             {
-                $responses[$artifact.name] = $respCode
+                $responses[$($packageMetadata.ArtifactName)] = $respCode
             }
         }
         else
         {
-            Write-Host "Pull request does not have any change for $($artifact.name). Skipping API change detect."
+            Write-Host "Pull request does not have any change for $($packageMetadata.ArtifactName)). Skipping API change detect."
         }
     }
     else
     {
-        Write-Host "No package is found in artifact path to find API changes for $($artifact.name)"
+        Write-Host "No package is found in artifact path to find API changes for $($packageMetadata.ArtifactName)"
     }
 }
 

eng/common/scripts/Generate-PR-Diff.ps1

@@ -19,14 +19,14 @@ Param (
   [string] $TargetPath
 )
 
-. (Join-Path $PSScriptRoot "Helpers" git-helpers.ps1)
+. (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1")
 
 function Get-ChangedServices {
     Param (
         [Parameter(Mandatory=$True)]
         [string[]] $ChangedFiles
     )
-    
+
     $changedServices = $ChangedFiles | Foreach-Object { if ($_ -match "sdk/([^/]+)") { $matches[1] } } | Sort-Object -Unique
 
     return $changedServices

eng/common/scripts/Package-Properties.ps1

@@ -15,6 +15,7 @@ class PackageProps
     [boolean]$IsNewSdk
     [string]$ArtifactName
     [string]$ReleaseStatus
+    [string[]]$DependentPackages
 
     PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory)
     {
@@ -55,7 +56,7 @@ class PackageProps
             if ($changeLogEntry -and $changeLogEntry.ReleaseStatus)
             {
               $this.ReleaseStatus = $changeLogEntry.ReleaseStatus.Trim().Trim("()")
-            } 
+            }
         }
         else
         {
@@ -101,7 +102,7 @@ function Get-PkgProperties
         return $pkgProps[0]
     }
 
-    LogError "Failed to retrive Properties for [$PackageName]"
+    LogError "Failed to retrieve Properties for [$PackageName]"
     return $null
 }
 
@@ -112,20 +113,35 @@ function Get-PrPkgProperties([string]$InputDiffJson) {
     $diff = Get-Content $InputDiffJson | ConvertFrom-Json
     $targetedFiles = $diff.ChangedFiles
 
-    foreach($pkg in $allPackageProperties)
+    $dependentPackagesForInclusion = @()
+    $lookup = @{}
+
+    foreach ($pkg in $allPackageProperties)
     {
         $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)"
+        $lookupKey = ($pkg.DirectoryPath).Replace($RepoRoot, "").SubString(1)
+        $lookup[$lookupKey] = $pkg
 
-        foreach($file in $targetedFiles)
+        foreach ($file in $targetedFiles)
         {
             $filePath = Resolve-Path (Join-Path $RepoRoot $file)
             $shouldInclude = $filePath -like "$pkgDirectory*"
             if ($shouldInclude) {
                 $packagesWithChanges += $pkg
+
+                if ($pkg.DependentPackages) {
+                    $dependentPackagesForInclusion += $pkg.DependentPackages
+                }
             }
         }
     }
 
+    foreach ($addition in $dependentPackagesForInclusion) {
+        if ($lookup[$addition]) {
+            $packagesWithChanges += $lookup[$addition]
+        }
+    }
+
     return $packagesWithChanges
 }
 

eng/common/scripts/Verify-ChangeLogs.ps1

@@ -0,0 +1,31 @@
+# Wrapper Script for ChangeLog Verification in a PR
+[CmdletBinding()]
+param (
+  [String]$PackagePropertiesFolder
+)
+Set-StrictMode -Version 3
+
+. (Join-Path $PSScriptRoot common.ps1)
+
+# find which packages we need to confirm the changelog for
+$packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json
+
+# grab the json file, then confirm the changelog entry for it
+$allPassing = $true
+foreach($propertiesFile in $packageProperties) {
+  $PackageProp = Get-Content -Path $propertiesFile | ConvertFrom-Json
+
+  $validChangeLog =  Confirm-ChangeLogEntry -ChangeLogLocation $PackageProp.ChangeLogPath -VersionString $PackageProp.Version -ForRelease $false
+
+  if (-not $validChangeLog) {
+    $allPassing = $false
+  }
+}
+
+
+if (!$allPassing)
+{
+  exit 1
+}
+
+exit 0