Sync eng/common directory with azure-sdk-tools for PR 3267 (#3655)

* Use seperate scripts

* address comments.

* do compare and update

* save on the service level readme

* have the helper for reuse function

* remove mgmt table

* changes

* fix

* no return on error

* return if no contents

* Address comments

* change the table

* address wes comments.

* address wes comments.

* address more comments.

Co-authored-by: sima-zhu <sizhu@microsoft.com>

eng/common/scripts/Helpers/Metadata-Helpers.ps1

@@ -80,3 +80,56 @@ function GetPrimaryCodeOwner ([string]$TargetDirectory)
     Write-Warning "No code owner found in $TargetDirectory."
     return $null
 }
+
+function GetDocsMsService($packageInfo, $serviceName) 
+{
+  $service = $serviceName.ToLower().Replace(' ', '').Replace('/', '-')
+  if ($packageInfo.MSDocService) {
+    # Use MSDocService in csv metadata to override the service directory    
+    # TODO: Use taxonomy for service name -- https://github.com/Azure/azure-sdk-tools/issues/1442
+    $service = $packageInfo.MSDocService
+  }
+  Write-Host "The service of package: $service"
+  return $service
+}
+
+function GenerateDocsMsMetadata($language, $langTitle = "", $serviceName, $tenantId, $clientId, $clientSecret, $msService) 
+{
+  if (!$langTitle) {
+    $langTitle = "Azure $serviceName SDK for $language"
+  }
+  $langDescription = "Reference for Azure $serviceName SDK for $language"
+  # Github url for source code: e.g. https://github.com/Azure/azure-sdk-for-js
+  $serviceBaseName = $serviceName.ToLower().Replace(' ', '').Replace('/', '-')
+  $author = GetPrimaryCodeOwner -TargetDirectory "/sdk/$serviceBaseName/"
+  $msauthor = ""
+  if (!$author) {
+    LogError "Cannot fetch the author from CODEOWNER file."
+  }
+  elseif ($TenantId -and $ClientId -and $ClientSecret) {
+    $msauthor = GetMsAliasFromGithub -TenantId $tenantId -ClientId $clientId -ClientSecret $clientSecret -GithubUser $author
+  }
+  # Default value
+  if (!$msauthor) {
+    LogError "No ms.author found for $author. "
+    $msauthor = $author
+  }
+  $date = Get-Date -Format "MM/dd/yyyy"
+  $header = @"
+---
+title: $langTitle
+description: $langDescription
+author: $author
+ms.author: $msauthor
+ms.date: $date
+ms.topic: reference
+ms.devlang: $language
+ms.service: $msService
+---
+"@
+  return $header
+}
+
+function ServiceLevelReadmeNameStyle($serviceName) {
+  return $serviceName.ToLower().Replace(' ', '-').Replace('/', '-')
+}

eng/common/scripts/Helpers/Package-Helpers.ps1

@@ -0,0 +1,37 @@
+function GetPackageKey($pkg) {
+    $pkgKey = $pkg.Package
+    $groupId = $null
+  
+    if ($pkg.PSObject.Members.Name -contains "GroupId") {
+      $groupId = $pkg.GroupId
+    }
+  
+    if ($groupId) {
+      $pkgKey = "${groupId}:${pkgKey}"
+    }
+  
+    return $pkgKey
+  }
+  
+  # Different language needs a different way to index the package. Build a map in convienice to lookup the package.
+  # E.g. <groupId>:<packageName> is the package key in java.
+  function GetPackageLookup($packageList) {
+    $packageLookup = @{}
+  
+    foreach ($pkg in $packageList) {
+      $pkgKey = GetPackageKey $pkg
+  
+      # We want to prefer updating non-hidden packages but if there is only
+      # a hidden entry then we will return that
+      if (!$packageLookup.ContainsKey($pkgKey) -or $packageLookup[$pkgKey].Hide -eq "true") {
+        $packageLookup[$pkgKey] = $pkg
+      }
+      else {
+        # Warn if there are more then one non-hidden package
+        if ($pkg.Hide -ne "true") {
+          Write-Host "Found more than one package entry for $($pkg.Package) selecting the first non-hidden one."
+        }
+      }
+    }
+    return $packageLookup
+  }

eng/common/scripts/Service-Level-Readme-Automation.ps1

@@ -0,0 +1,236 @@
+<#
+.SYNOPSIS
+The script is to generate service level readme if it is missing. 
+For exist ones, we do 2 things here:
+1. Generate the client but not import to the existing service level readme.
+2. Update the metadata of service level readme
+
+.DESCRIPTION
+Given a doc repo location, and the credential for fetching the ms.author. 
+Generate missing service level readme and updating metadata of the existing ones.
+
+.PARAMETER DocRepoLocation
+Location of the documentation repo. This repo may be sparsely checked out
+depending on the requirements for the domain
+
+.PARAMETER TenantId
+The aad tenant id/object id for ms.author.
+
+.PARAMETER ClientId
+The add client id/application id for ms.author.
+
+.PARAMETER ClientSecret
+The client secret of add app for ms.author.
+#>
+
+param(
+  [Parameter(Mandatory = $true)]
+  [string] $DocRepoLocation,
+
+  [Parameter(Mandatory = $false)]
+  [string]$TenantId,
+
+  [Parameter(Mandatory = $false)]
+  [string]$ClientId,
+
+  [Parameter(Mandatory = $false)]
+  [string]$ClientSecret
+)
+. $PSScriptRoot/common.ps1
+. $PSScriptRoot/Helpers/Metadata-Helpers.ps1
+. $PSScriptRoot/Helpers/Package-Helpers.ps1
+
+Set-StrictMode -Version 3
+
+function create-metadata-table($readmeFolder, $readmeName, $moniker, $msService, $clientTableLink, $mgmtTableLink, $serviceName)
+{
+  $readmePath = Join-Path $readmeFolder -ChildPath $readmeName
+  $content = ""  
+  if (Test-Path (Join-Path $readmeFolder -ChildPath $clientTableLink)) {
+    $content = "## Client packages - $moniker`r`n"
+    $content += "[!INCLUDE [client-packages]($clientTableLink)]`r`n"
+  }
+  if (Test-Path (Join-Path $readmeFolder -ChildPath $mgmtTableLink)) {
+    $content = "## Management packages - $moniker`r`n"
+    $content += "[!INCLUDE [mgmt-packages]($mgmtTableLink)]`r`n"
+  }
+  if (!$content) {
+    return
+  }
+  $null = New-Item -Path $readmePath -Force
+  $lang = $LanguageDisplayName
+  $langTitle = "Azure $serviceName SDK for $lang"
+  # Generate the front-matter for docs needs
+  $metadataString = GenerateDocsMsMetadata -language $lang -langTitle $langTitle -serviceName $serviceName `
+    -tenantId $TenantId -clientId $ClientId -clientSecret $ClientSecret `
+    -msService $msService
+  Add-Content -Path $readmePath -Value $metadataString
+
+  # Add tables, seperate client and mgmt.
+  $readmeHeader = "# $langTitle - $moniker"
+  Add-Content -Path $readmePath -Value $readmeHeader
+  Add-Content -Path $readmePath -Value $content
+}
+
+function compare-and-merge-metadata ($original, $updated) {
+  $originalTable = ConvertFrom-StringData -StringData $original -Delimiter ":"
+  $updatedTable = ConvertFrom-StringData -StringData $updated -Delimiter ":"
+  foreach ($key in $originalTable.Keys) {
+    if (!($updatedTable.ContainsKey($key))) {
+      Write-Warning "New metadata missed the entry: $key. Adding back."
+      $updated += "$key`: $($originalTable[$key])`r`n"
+    }
+  }
+  return $updated
+}
+
+# Update the metadata table.
+function update-metadata-table($readmeFolder, $readmeName, $serviceName, $msService)
+{
+  $readmePath = Join-Path $readmeFolder -ChildPath $readmeName
+  $readmeContent = Get-Content -Path $readmePath -Raw
+  $null = $readmeContent -match "---`n*(?<metadata>(.*`n)*)---`n*(?<content>(.*`n)*)"
+  $restContent = $Matches["content"]
+  $lang = $LanguageDisplayName
+  $orignalMetadata = $Matches["metadata"]
+  $metadataString = GenerateDocsMsMetadata -language $lang -serviceName $serviceName `
+    -tenantId $TenantId -clientId $ClientId -clientSecret $ClientSecret `
+    -msService $msService
+  $null = $metadataString -match "---`n*(?<metadata>(.*`n)*)---"
+  $mergedMetadata = compare-and-merge-metadata -original $orignalMetadata -updated $Matches["metadata"]
+  Set-Content -Path $readmePath -Value "---`n$mergedMetadata---`n$restContent" -NoNewline
+}
+
+function generate-markdown-table($readmeFolder, $readmeName, $packageInfo, $moniker) {
+  $tableHeader = "| Reference | Package | Source |`r`n|---|---|---|`r`n" 
+  $tableContent = ""
+  # Here is the table, the versioned value will
+  foreach ($pkg in $packageInfo) {
+    $repositoryLink = $RepositoryUri
+    $packageLevelReadme = ""
+    if (Test-Path "Function:$GetPackageLevelReadmeFn") {
+      $packageLevelReadme = &$GetPackageLevelReadmeFn -packageMetadata $pkg
+    }
+    
+    $referenceLink = "[$($pkg.DisplayName)]($packageLevelReadme-readme.md)"
+    if (!(Test-Path (Join-Path $readmeFolder -ChildPath "$packageLevelReadme-readme.md"))) {
+      $referenceLink = $pkg.DisplayName
+    }
+    $githubLink = $GithubUri
+    if ($pkg.PSObject.Members.Name -contains "DirectoryPath") {
+      $githubLink = "$GithubUri/blob/main/$($pkg.DirectoryPath)"
+    }
+    $line = "|$referenceLink|[$($pkg.Package)]($repositoryLink/$($pkg.Package))|[Github]($githubLink)|`r`n"
+    $tableContent += $line
+  }
+  $readmePath = Join-Path $readmeFolder -ChildPath $readmeName
+  if($tableContent) {
+    $null = New-Item -Path $readmePath -ItemType File -Force
+    Add-Content -Path $readmePath -Value $tableHeader -NoNewline
+    Add-Content -Path $readmePath -Value $tableContent -NoNewline
+  }
+}
+
+function generate-service-level-readme($readmeBaseName, $pathPrefix, $packageInfos, $serviceName, $moniker) {
+  # Add ability to override
+  # Fetch the service readme name
+  $msService = GetDocsMsService -packageInfo $packageInfos[0] -serviceName $serviceName
+
+  $readmeFolder = "$DocRepoLocation/$pathPrefix/$moniker/"
+  $serviceReadme = "$readmeBaseName.md"
+  $clientIndexReadme  = "$readmeBaseName-client-index.md"
+  $mgmtIndexReadme  = "$readmeBaseName-mgmt-index.md"
+  $clientPackageInfo = $packageInfos.Where({ 'client' -eq $_.Type }) | Sort-Object -Property Package
+  if ($clientPackageInfo) {
+    generate-markdown-table -readmeFolder $readmeFolder -readmeName "$clientIndexReadme" -packageInfo $clientPackageInfo -moniker $moniker
+  }
+  # TODO: we currently do not have the right decision on how we display mgmt packages. Will track the mgmt work in issue. 
+  # https://github.com/Azure/azure-sdk-tools/issues/3422
+  # $mgmtPackageInfo = $packageInfos.Where({ 'mgmt' -eq $_.Type }) | Sort-Object -Property Package
+  # if ($mgmtPackageInfo) {
+  #   generate-markdown-table -readmeFolder $readmeFolder -readmeName "$mgmtIndexReadme" -packageInfo $mgmtPackageInfo -moniker $moniker
+  # }
+  if (!(Test-Path (Join-Path $readmeFolder -ChildPath $serviceReadme))) {
+    create-metadata-table -readmeFolder $readmeFolder -readmeName $serviceReadme -moniker $moniker -msService $msService `
+      -clientTableLink $clientIndexReadme -mgmtTableLink $mgmtIndexReadme `
+      -serviceName $serviceName
+  }
+  else {
+    update-metadata-table -readmeFolder $readmeFolder -readmeName $serviceReadme -serviceName $serviceName -msService $msService
+  }
+}
+
+$fullMetadata = Get-CSVMetadata
+$monikers = @("latest", "preview")
+foreach($moniker in $monikers) {
+  # The onboarded packages return is key-value pair, which key is the package index, and value is the package info from {metadata}.json
+  # E.g. 
+  # Key as: @azure/storage-blob
+  # Value as: 
+  # {
+  #   "Name": "@azure/storage-blob",
+  #   "Version": "12.10.0-beta.1",
+  #   "DevVersion": null,
+  #   "DirectoryPath": "sdk/storage/storage-blob",
+  #   "ServiceDirectory": "storage",
+  #   "ReadMePath": "sdk/storage/storage-blob/README.md",
+  #   "ChangeLogPath": "sdk/storage/storage-blob/CHANGELOG.md",
+  #   "Group": null,
+  #   "SdkType": "client",
+  #   "IsNewSdk": true,
+  #   "ArtifactName": "azure-storage-blob",
+  #   "ReleaseStatus": "2022-04-19"
+  # }
+  $onboardedPackages = &$GetOnboardedDocsMsPackagesForMonikerFn `
+    -DocRepoLocation $DocRepoLocation -moniker $moniker
+  $csvMetadata = @()
+  foreach($metadataEntry in $fullMetadata) {
+    if ($metadataEntry.Package -and $metadataEntry.Hide -ne 'true') {
+      $pkgKey = GetPackageKey $metadataEntry
+      if($onboardedPackages.ContainsKey($pkgKey)) {
+        if ($onboardedPackages[$pkgKey] -and $onboardedPackages[$pkgKey].DirectoryPath) {
+          if (!($metadataEntry.PSObject.Members.Name -contains "DirectoryPath")) {
+            Add-Member -InputObject $metadataEntry `
+              -MemberType NoteProperty `
+              -Name DirectoryPath `
+              -Value $onboardedPackages[$pkgKey].DirectoryPath
+          }
+        }
+        $csvMetadata += $metadataEntry
+      }
+    }
+  }
+  $packagesForService = @{}
+  $allPackages = GetPackageLookup $csvMetadata
+  foreach ($metadataKey in $allPackages.Keys) {
+    $metadataEntry = $allPackages[$metadataKey]
+    if (!$metadataEntry.ServiceName) {
+      LogWarning "Empty ServiceName for package `"$metadataKey`". Skipping."
+      continue
+    }
+    $packagesForService[$metadataKey] = $metadataEntry
+  }
+  $services = @{}
+  foreach ($package in $packagesForService.Values) {
+    if ($package.ServiceName -eq 'Other') {
+      # Skip packages under the service category "Other". Those will be handled
+      # later
+      continue
+    }
+    if (!$services.ContainsKey($package.ServiceName)) {
+      $services[$package.ServiceName] = $true
+    }
+  }
+  foreach ($service in $services.Keys) {
+    Write-Host "Building service: $service"
+    
+    $servicePackages = $packagesForService.Values.Where({ $_.ServiceName -eq $service })
+  
+  
+    $serviceReadmeBaseName = ServiceLevelReadmeNameStyle -serviceName $service
+    $hrefPrefix = "docs-ref-services"
+  
+    generate-service-level-readme -readmeBaseName $serviceReadmeBaseName -pathPrefix $hrefPrefix `
+      -packageInfos $servicePackages -serviceName $service -moniker $moniker
+  }
+}

eng/common/scripts/common.ps1

@@ -50,6 +50,8 @@ $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview"
 $TestProxyTrustCertFn = "Import-Dev-Cert-${Language}"
 $ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages"
 $GetOnboardedDocsMsPackagesFn = "Get-${Language}-OnboardedDocsMsPackages"
+$GetOnboardedDocsMsPackagesForMonikerFn = "Get-${Language}-OnboardedDocsMsPackagesForMoniker"
 $GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
 $GetDocsMsTocChildrenForManagementPackagesFn = "Get-${Language}-DocsMsTocChildrenForManagementPackages"
 $UpdateDocsMsTocFn = "Get-${Language}-UpdatedDocsMsToc"
+$GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme"