Centralize the docindex scripts and regenerate Github IO landing page. (#892)

* Centralize the docindex scripts and regenerate Github IO landing page.

eng/docs/index/Generate-DocIndex.ps1

@@ -1,68 +1,150 @@
 # Generates an index page for cataloging different versions of the Docs
-
 [CmdletBinding()]
 Param (
+    $DocFx,
     $RepoRoot,
     $DocGenDir
 )
+. "${PSScriptRoot}\..\..\common\scripts\common.ps1"
+$GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex"
 
-$ServiceMapping = @{
+# Given the metadata url under https://github.com/Azure/azure-sdk/tree/master/_data/releases/latest, 
-    "core" = "Core";
+# the function will return the csv metadata back as part of response.
-    "storage" = "Storage";
+function Get-CSVMetadata ([string]$MetadataUri) {
+    $metadataResponse = Invoke-RestMethod -Uri $MetadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv
+    return $metadataResponse
+}
+  
+# Given the github io blob storage url and language regex,
+# the helper function will return a list of artifact names.
+function Get-BlobStorage-Artifacts($blobStorageUrl, $blobDirectoryRegex, $blobArtifactsReplacement) {
+    LogDebug "Reading artifact from storage blob ..."
+    $returnedArtifacts = @()
+    $pageToken = ""
+    Do {
+      $resp = ""
+      if (!$pageToken) {
+        # First page call.
+        $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrl
+      }
+      else {
+        # Next page call
+        $blobStorageUrlPageToken = $blobStorageUrl + "&marker=$pageToken"
+        $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrlPageToken
+      }
+      # Convert to xml documents. 
+      $xmlDoc = [xml](removeBomFromString $resp)
+      foreach ($elem in $xmlDoc.EnumerationResults.Blobs.BlobPrefix) {
+        # What service return like "dotnet/Azure.AI.Anomalydetector/", needs to fetch out "Azure.AI.Anomalydetector"
+        $artifact = $elem.Name -replace $blobDirectoryRegex, $blobArtifactsReplacement
+        $returnedArtifacts += $artifact
+      }
+      # Fetch page token
+      $pageToken = $xmlDoc.EnumerationResults.NextMarker
+    } while ($pageToken)
+    return $returnedArtifacts
+  }
+  
+# The sequence of Bom bytes differs by different encoding. 
+# The helper function here is only to strip the utf-8 encoding system as it is used by blob storage list api.
+# Return the original string if not in BOM utf-8 sequence.
+function RemoveBomFromString([string]$bomAwareString) {
+    if ($bomAwareString.length -le 3) {
+        return $bomAwareString
+    }
+    $bomPatternByteArray = [byte[]] (0xef, 0xbb, 0xbf)
+    # The default encoding for powershell is ISO-8859-1, so converting bytes with the encoding.
+    $bomAwareBytes = [Text.Encoding]::GetEncoding(28591).GetBytes($bomAwareString.Substring(0, 3))
+    if (@(Compare-Object $bomPatternByteArray $bomAwareBytes -SyncWindow 0).Length -eq 0) {
+        return $bomAwareString.Substring(3)
+    }
+    return $bomAwareString
+}
+  
+function Get-TocMapping { 
+    Param (
+        [Parameter(Mandatory = $true)] [Object[]] $metadata,
+        [Parameter(Mandatory = $true)] [String[]] $artifacts
+    )
+    # Used for sorting the toc display order
+    $orderServiceMapping = @{}
+
+    foreach ($artifact in $artifacts) {
+        $packageInfo = $metadata | ? {$_.Package -eq $artifact}
+        
+        if ($packageInfo -and $packageInfo[0].Hide -eq 'true') {
+            LogDebug "The artifact $artifact set 'Hide' to 'true'."
+            continue
+        }
+        $serviceName = ""
+        if (!$packageInfo -or !$packageInfo[0].ServiceName) {
+            LogWarning "There is no service name for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. "
+            # If no service name retrieved, print out warning message, and put it into Other page.
+            $serviceName = "Other"
+        }
+        else {
+            if ($packageInfo.Length -gt 1) {
+                LogWarning "There are more than 1 packages fetched out for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. "
+            }
+            $serviceName = $packageInfo[0].ServiceName.Trim()
+        }
+        $orderServiceMapping[$artifact] = $serviceName
+    }
+    return $orderServiceMapping                   
 }
 
-Write-Verbose "Name Reccuring paths with variable names"
+function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang) {
-$DocFxTool = "${RepoRoot}/docfx/docfx.exe"
+    LogDebug "Start generating the docfx toc and build docfx site..."
-$DocOutDir = "${RepoRoot}/docfx_project"
+    $DocOutDir = "${RepoRoot}/docfx_project"
 
-Write-Verbose "Initializing Default DocFx Site..."
+    LogDebug "Initializing Default DocFx Site..."
-& "${DocFxTool}" init -q -o "${DocOutDir}"
+    & $($DocFx) init -q -o "${DocOutDir}"
-
+    # The line below is used for testing in local
-Write-Verbose "Copying template and configuration..."
+    #docfx init -q -o "${DocOutDir}"
-New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory"
+    LogDebug "Copying template and configuration..."
-Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse
+    New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" -Force
-Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force
+    Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse
-
+    Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force
-Write-Verbose "Creating Index using service directory and package names from repo..."
+    $YmlPath = "${DocOutDir}/api"
-$ServiceList = Get-ChildItem "$($RepoRoot)/sdk" -Directory -Exclude template | Sort-Object
+    New-Item -Path $YmlPath -Name "toc.yml" -Force
-$YmlPath = "${DocOutDir}/api"
+    $visitedService = @{}
-New-Item -Path $YmlPath -Name "toc.yml" -Force
+    # Sort and display toc service name by alphabetical order, and then sort artifact by order.
-
+    foreach ($serviceMapping in ($tocContent.GetEnumerator() | Sort-Object Value, Key)) {
-Write-Verbose "Creating Index for client packages..."
+        $artifact = $serviceMapping.Key
-foreach ($Dir in $ServiceList)
+        $serviceName = $serviceMapping.Value
-{
+        $fileName = ($serviceName -replace '\s', '').ToLower().Trim()
-    New-Item -Path $YmlPath -Name "$($Dir.Name).md" -Force
+        if ($visitedService.ContainsKey($serviceName)) {
-    $ServiceName = If ($ServiceMapping.Contains($Dir.Name)) { $ServiceMapping[$Dir.Name] } Else { $Dir.Name }
+            Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact"
-    Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: $($ServiceName)`r`n  href: $($Dir.Name).md"
+        }
-    $PkgList = Get-ChildItem $Dir.FullName -Directory -Exclude .vs, .vscode, performance-stress
+        else {
-
+            Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: ${serviceName}`r`n  href: ${fileName}.md"
-    if (($PkgList | Measure-Object).count -eq 0)
+            New-Item -Path $YmlPath -Name "${fileName}.md" -Force
-    {
+            Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact"
-        continue
+            $visitedService[$serviceName] = $true
-    }
-    Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Client"
-    Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "---"
-    Write-Verbose "Operating on Client Packages for $($Dir.Name)"
-
-    foreach ($Pkg in $PkgList)
-    {
-        if (Test-Path "$($Pkg.FullName)\src")
-        {
-            Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $($Pkg.BaseName)"
         }
     }
+
+    # Generate toc homepage.
+    LogDebug "Creating Site Title and Navigation..."
+    New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force
+    Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for $lang APIs`r`n  href: api/`r`n  homepage: api/index.md"
+
+    LogDebug "Copying root markdowns"
+    Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force
+    Copy-Item "$($RepoRoot)/CONTRIBUTING.md" -Destination "${DocOutDir}/api/CONTRIBUTING.md" -Force
+
+    LogDebug "Building site..."
+    & $($DocFx) build "${DocOutDir}/docfx.json"
+    # The line below is used for testing in local
+    #docfx build "${DocOutDir}/docfx.json"
+    Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force    
 }
 
-Write-Verbose "Creating Site Title and Navigation..."
+if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $GetGithubIoDocIndexFn  }).Count -gt 0)
-New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force
+{
-Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for C++ APIs`r`n  href: api/`r`n  homepage: api/index.md"
+    &$GetGithubIoDocIndexFn
-
+}
-Write-Verbose "Copying root markdowns"
+else
-Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force
+{
-Copy-Item "$($RepoRoot)/CONTRIBUTING.md" -Destination "${DocOutDir}/api/CONTRIBUTING.md" -Force
+    LogWarning "The function '$GetGithubIoDocIndexFn' was not found."
-
+}
-Write-Verbose "Building site..."
-& "${DocFxTool}" build "${DocOutDir}/docfx.json"
-
-Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force
-Copy-Item "${DocGenDir}/assets/toc.yml" -Destination "${DocOutDir}/_site/" -Force

eng/pipelines/doc-index.yml

@@ -8,25 +8,23 @@ jobs:
         inputs:
           versionSpec: '3.6'
 
-      - template: /eng/common/pipelines/templates/steps/replace-relative-links.yml
-        parameters:
-          TargetFolder: '.'
-          RootFolder: '.'
-          BuildSHA: $(Build.SourceVersion)
-          RepoId: 'Azure/azure-sdk-for-cpp'
-
       - pwsh: |
           Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://github.com/dotnet/docfx/releases/download/v2.43.2/docfx.zip" `
           -OutFile "docfx.zip" | Wait-Process; Expand-Archive -Path "docfx.zip" -DestinationPath "./docfx/"
+          echo "##vso[task.setvariable variable=docfxPath;isOutput=true]$(Build.SourcesDirectory)/docfx/docfx.exe"
         workingDirectory: $(Build.SourcesDirectory)
         displayName: Download and Extract DocFX
-
+        name: setupDocfxTool
-      - pwsh: >-
+      - task: PowerShell@2
-          $(Build.SourcesDirectory)/eng/docs/index/Generate-DocIndex.ps1
+        displayName: "Generate Doc Index"
-          -RepoRoot $(Build.SourcesDirectory)
+        inputs:
-          -DocGenDir "$(Build.SourcesDirectory)/eng/docs/index"
+          pwsh: true
-          -verbose
+          filePath: $(Build.SourcesDirectory)/eng/docs/index/Generate-DocIndex.ps1
-        displayName: 'Generate Doc Index'
+          arguments: >
+            -Docfx $(setupDocfxTool.docfxPath)
+            -RepoRoot $(Build.SourcesDirectory)
+            -DocGenDir "$(Build.SourcesDirectory)/eng/docs/index"
+            -verbose
 
       - task: UsePythonVersion@0
         displayName: 'Use Python 3.6'

eng/scripts/Language-Settings.ps1

@@ -1,8 +1,8 @@
 $Language = "cpp"
 $PackageRepository = "CPP"
 $packagePattern = "*.json"
-$MetadataUri = ""
+$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/cpp-packages.csv"
-
+$BlobStorageUrl = "https://azuresdkdocs.blob.core.windows.net/%24web?restype=container&comp=list&prefix=cpp%2F&delimiter=%2F"
 
 # Parse out package publishing information given a nupkg ZIP format.
 function Get-cpp-PackageInfoFromPackageFile($pkg, $workingDirectory) 
@@ -45,4 +45,15 @@ function Publish-cpp-GithubIODocs ($DocLocation, $PublicArtifactLocation)
   $packageInfo = (Get-Content (Join-Path $DocLocation 'package-info.json') | ConvertFrom-Json)
   $releaseTag = RetrieveReleaseTag "CPP" $PublicArtifactLocation
   Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version -ReleaseTag $releaseTag
+}
+
+function Get-cpp-GithubIoDocIndex() {
+  # Fetch out all package metadata from csv file.
+  $metadata = Get-CSVMetadata -MetadataUri $MetadataUri
+  # Get the artifacts name from blob storage
+  $artifacts =  Get-BlobStorage-Artifacts -blobStorageUrl $BlobStorageUrl -blobDirectoryRegex "^cpp/(.*)/$" -blobArtifactsReplacement '$1'
+  # Build up the artifact to service name mapping for GithubIo toc.
+  $tocContent = Get-TocMapping -metadata $metadata -artifacts $artifacts
+  # Generate yml/md toc files and build site.
+  GenerateDocfxTocContent -tocContent $tocContent -lang "C++"
 }