Sync eng/common directory with azure-sdk-tools for PR 2327 (#3158)

* Use docker validation on release pipeline

* format

* Update Update-DocsMsMetadata.ps1

* Update Update-DocsMsMetadata.ps1

* Update Update-DocsMsMetadata.ps1

* For testing null purpose

* Update eng/common/scripts/Update-DocsMsMetadata.ps1

* Update eng/common/scripts/Update-DocsMsMetadata.ps1

Co-authored-by: Ben Broderick Phillips <ben@benbp.net>

Co-authored-by: sima-zhu <sizhu@microsoft.com>
Co-authored-by: Sima Zhu <48036328+sima-zhu@users.noreply.github.com>
Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>
Co-authored-by: Ben Broderick Phillips <ben@benbp.net>

eng/common/pipelines/templates/steps/update-docsms-metadata.yml

@@ -26,7 +26,12 @@ parameters:
     type: object
     default:
       - '**'
-
+  - name: PackageSourceOverride
+    type: string
+    default: ''
+  - name: DocValidationImageId
+    type: string
+    default: ''
 steps:
 - template: /eng/common/pipelines/templates/steps/enable-long-path-support.yml
 
@@ -73,7 +78,13 @@ steps:
     parameters:
       WorkingDirectory: $(DocRepoLocation)
       DefaultBranchVariableName: TargetBranchName
-
+# Pull and build the docker image.
+- ${{ if ne(parameters.DocValidationImageId, '') }}:
+  - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml
+    parameters:
+      ContainerRegistryClientId: $(azuresdkimages-cr-clientid)
+      ContainerRegistryClientSecret: $(azuresdkimages-cr-clientsecret)
+      ImageId: '${{ parameters.DocValidationImageId }}'
 - pwsh: |
     $packageInfoJson = '${{ convertToJson(parameters.PackageInfoLocations) }}'.Trim('"').Replace("\\", "/")
     $packageInfoLocations = ConvertFrom-Json $packageInfoJson
@@ -81,7 +92,9 @@ steps:
       -PackageInfoJsonLocations $packageInfoLocations `
       -DocRepoLocation "$(DocRepoLocation)" `
       -Language '${{parameters.Language}}' `
-      -RepoId '${{ parameters.RepoId }}'
+      -RepoId '${{ parameters.RepoId }}' `
+      -DocValidationImageId '${{ parameters.DocValidationImageId }}' `
+      -PackageSourceOverride '${{ parameters.PackageSourceOverride }}'
   displayName: Apply Documentation Updates
 
 - template: /eng/common/pipelines/templates/steps/git-push-changes.yml

eng/common/scripts/Update-DocsMsMetadata.ps1

@@ -28,6 +28,15 @@ Programming language to supply to metadata
 .PARAMETER RepoId
 GitHub repository ID of the SDK. Typically of the form: 'Azure/azure-sdk-for-js'
 
+.PARAMETER DocValidationImageId
+The docker image id in format of '$containerRegistry/$imageName:$tag'
+e.g. azuresdkimages.azurecr.io/jsrefautocr:latest
+
+.PARAMETER PackageSourceOverride
+Optional parameter to supply a different package source (useful for daily dev
+docs generation from pacakges which are not published to the default feed). This
+variable is meant to be used in the domain-specific business logic in
+&$ValidateDocsMsPackagesFn
 #>
 
 param(
@@ -41,7 +50,13 @@ param(
   [string]$Language,
 
   [Parameter(Mandatory = $true)]
-  [string]$RepoId
+  [string]$RepoId,
+
+  [Parameter(Mandatory = $false)]
+  [string]$DocValidationImageId,
+
+  [Parameter(Mandatory = $false)]
+  [string]$PackageSourceOverride
 )
 
 . (Join-Path $PSScriptRoot common.ps1)
@@ -104,13 +119,12 @@ ms.technology: azure
 ms.devlang: $Language
 ms.service: $service
 ---
-
 "@
 
   return "$header`n$ReadmeContent"
 }
 
-function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) { 
+function GetPackageInfoJson ($packageInfoJsonLocation) {
   if (!(Test-Path $packageInfoJsonLocation)) {
     LogWarning "Package metadata not found for $packageInfoJsonLocation"
     return
@@ -118,7 +132,6 @@ function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) {
   
   $packageInfoJson = Get-Content $packageInfoJsonLocation -Raw
   $packageInfo = ConvertFrom-Json $packageInfoJson
-  $originalVersion = [AzureEngSemanticVersion]::ParseVersionString($packageInfo.Version)
   if ($packageInfo.DevVersion) {
     # If the package is of a dev version there may be language-specific needs to 
     # specify the appropriate version. For example, in the case of JS, the dev 
@@ -131,6 +144,11 @@ function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) {
       $packageInfo.Version = $packageInfo.DevVersion
     }
   }
+  return $packageInfo
+}
+
+function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation, $packageInfo) { 
+  $originalVersion = [AzureEngSemanticVersion]::ParseVersionString($packageInfo.Version)
 
   $packageMetadataArray = (Get-CSVMetadata).Where({ $_.Package -eq $packageInfo.Name -and $_.GroupId -eq $packageInfo.Group -and $_.Hide -ne 'true' -and $_.New -eq 'true' })
   if ($packageMetadataArray.Count -eq 0) { 
@@ -176,7 +194,18 @@ function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) {
     -Value $packageInfoJson
 }
 
-foreach ($packageInfo in $PackageInfoJsonLocations) {
+foreach ($packageInfoLocation in $PackageInfoJsonLocations) {
   Write-Host "Updating metadata for package: $packageInfo"
-  UpdateDocsMsMetadataForPackage $packageInfo
+
+  # Convert package metadata json file to metadata json property.
+  $packageInfo = GetPackageInfoJson $packageInfoLocation
+  # Add validation step for daily update and release
+  if ($ValidateDocsMsPackagesFn -and (Test-Path "Function:$ValidateDocsMsPackagesFn")) {
+    &$ValidateDocsMsPackagesFn -PackageInfo $packageInfo -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $DocValidationImageId
+    if ($LASTEXITCODE) {
+      LogError "The package failed Doc.Ms validation. Check https://aka.ms/azsdk/docs/docker for more details on how to diagnose this issue."
+      exit $LASTEXITCODE
+    }
+  }
+  UpdateDocsMsMetadataForPackage $packageInfoLocation $packageInfo
 }

eng/common/scripts/common.ps1

@@ -45,3 +45,4 @@ $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageS
 $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex"
 $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview"
 $TestProxyTrustCertFn = "Import-Dev-Cert-${Language}"
+$ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages"