diff --git a/eng/docs/index/Generate-DocIndex.ps1 b/eng/docs/index/Generate-DocIndex.ps1 index 779e29b20..0fc146a89 100644 --- a/eng/docs/index/Generate-DocIndex.ps1 +++ b/eng/docs/index/Generate-DocIndex.ps1 @@ -1,68 +1,150 @@ # Generates an index page for cataloging different versions of the Docs - [CmdletBinding()] Param ( + $DocFx, $RepoRoot, $DocGenDir ) +. "${PSScriptRoot}\..\..\common\scripts\common.ps1" +$GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex" -$ServiceMapping = @{ - "core" = "Core"; - "storage" = "Storage"; +# Given the metadata url under https://github.com/Azure/azure-sdk/tree/master/_data/releases/latest, +# the function will return the csv metadata back as part of response. +function Get-CSVMetadata ([string]$MetadataUri) { + $metadataResponse = Invoke-RestMethod -Uri $MetadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv + return $metadataResponse +} + +# Given the github io blob storage url and language regex, +# the helper function will return a list of artifact names. +function Get-BlobStorage-Artifacts($blobStorageUrl, $blobDirectoryRegex, $blobArtifactsReplacement) { + LogDebug "Reading artifact from storage blob ..." + $returnedArtifacts = @() + $pageToken = "" + Do { + $resp = "" + if (!$pageToken) { + # First page call. + $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrl + } + else { + # Next page call + $blobStorageUrlPageToken = $blobStorageUrl + "&marker=$pageToken" + $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrlPageToken + } + # Convert to xml documents. + $xmlDoc = [xml](removeBomFromString $resp) + foreach ($elem in $xmlDoc.EnumerationResults.Blobs.BlobPrefix) { + # What service return like "dotnet/Azure.AI.Anomalydetector/", needs to fetch out "Azure.AI.Anomalydetector" + $artifact = $elem.Name -replace $blobDirectoryRegex, $blobArtifactsReplacement + $returnedArtifacts += $artifact + } + # Fetch page token + $pageToken = $xmlDoc.EnumerationResults.NextMarker + } while ($pageToken) + return $returnedArtifacts + } + +# The sequence of Bom bytes differs by different encoding. +# The helper function here is only to strip the utf-8 encoding system as it is used by blob storage list api. +# Return the original string if not in BOM utf-8 sequence. +function RemoveBomFromString([string]$bomAwareString) { + if ($bomAwareString.length -le 3) { + return $bomAwareString + } + $bomPatternByteArray = [byte[]] (0xef, 0xbb, 0xbf) + # The default encoding for powershell is ISO-8859-1, so converting bytes with the encoding. + $bomAwareBytes = [Text.Encoding]::GetEncoding(28591).GetBytes($bomAwareString.Substring(0, 3)) + if (@(Compare-Object $bomPatternByteArray $bomAwareBytes -SyncWindow 0).Length -eq 0) { + return $bomAwareString.Substring(3) + } + return $bomAwareString +} + +function Get-TocMapping { + Param ( + [Parameter(Mandatory = $true)] [Object[]] $metadata, + [Parameter(Mandatory = $true)] [String[]] $artifacts + ) + # Used for sorting the toc display order + $orderServiceMapping = @{} + + foreach ($artifact in $artifacts) { + $packageInfo = $metadata | ? {$_.Package -eq $artifact} + + if ($packageInfo -and $packageInfo[0].Hide -eq 'true') { + LogDebug "The artifact $artifact set 'Hide' to 'true'." + continue + } + $serviceName = "" + if (!$packageInfo -or !$packageInfo[0].ServiceName) { + LogWarning "There is no service name for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. " + # If no service name retrieved, print out warning message, and put it into Other page. + $serviceName = "Other" + } + else { + if ($packageInfo.Length -gt 1) { + LogWarning "There are more than 1 packages fetched out for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. " + } + $serviceName = $packageInfo[0].ServiceName.Trim() + } + $orderServiceMapping[$artifact] = $serviceName + } + return $orderServiceMapping } -Write-Verbose "Name Reccuring paths with variable names" -$DocFxTool = "${RepoRoot}/docfx/docfx.exe" -$DocOutDir = "${RepoRoot}/docfx_project" +function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang) { + LogDebug "Start generating the docfx toc and build docfx site..." + $DocOutDir = "${RepoRoot}/docfx_project" -Write-Verbose "Initializing Default DocFx Site..." -& "${DocFxTool}" init -q -o "${DocOutDir}" - -Write-Verbose "Copying template and configuration..." -New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" -Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse -Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force - -Write-Verbose "Creating Index using service directory and package names from repo..." -$ServiceList = Get-ChildItem "$($RepoRoot)/sdk" -Directory -Exclude template | Sort-Object -$YmlPath = "${DocOutDir}/api" -New-Item -Path $YmlPath -Name "toc.yml" -Force - -Write-Verbose "Creating Index for client packages..." -foreach ($Dir in $ServiceList) -{ - New-Item -Path $YmlPath -Name "$($Dir.Name).md" -Force - $ServiceName = If ($ServiceMapping.Contains($Dir.Name)) { $ServiceMapping[$Dir.Name] } Else { $Dir.Name } - Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: $($ServiceName)`r`n href: $($Dir.Name).md" - $PkgList = Get-ChildItem $Dir.FullName -Directory -Exclude .vs, .vscode, performance-stress - - if (($PkgList | Measure-Object).count -eq 0) - { - continue - } - Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Client" - Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "---" - Write-Verbose "Operating on Client Packages for $($Dir.Name)" - - foreach ($Pkg in $PkgList) - { - if (Test-Path "$($Pkg.FullName)\src") - { - Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $($Pkg.BaseName)" + LogDebug "Initializing Default DocFx Site..." + & $($DocFx) init -q -o "${DocOutDir}" + # The line below is used for testing in local + #docfx init -q -o "${DocOutDir}" + LogDebug "Copying template and configuration..." + New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" -Force + Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse + Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force + $YmlPath = "${DocOutDir}/api" + New-Item -Path $YmlPath -Name "toc.yml" -Force + $visitedService = @{} + # Sort and display toc service name by alphabetical order, and then sort artifact by order. + foreach ($serviceMapping in ($tocContent.GetEnumerator() | Sort-Object Value, Key)) { + $artifact = $serviceMapping.Key + $serviceName = $serviceMapping.Value + $fileName = ($serviceName -replace '\s', '').ToLower().Trim() + if ($visitedService.ContainsKey($serviceName)) { + Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact" + } + else { + Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: ${serviceName}`r`n href: ${fileName}.md" + New-Item -Path $YmlPath -Name "${fileName}.md" -Force + Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact" + $visitedService[$serviceName] = $true } } + + # Generate toc homepage. + LogDebug "Creating Site Title and Navigation..." + New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force + Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for $lang APIs`r`n href: api/`r`n homepage: api/index.md" + + LogDebug "Copying root markdowns" + Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force + Copy-Item "$($RepoRoot)/CONTRIBUTING.md" -Destination "${DocOutDir}/api/CONTRIBUTING.md" -Force + + LogDebug "Building site..." + & $($DocFx) build "${DocOutDir}/docfx.json" + # The line below is used for testing in local + #docfx build "${DocOutDir}/docfx.json" + Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force } -Write-Verbose "Creating Site Title and Navigation..." -New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force -Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for C++ APIs`r`n href: api/`r`n homepage: api/index.md" - -Write-Verbose "Copying root markdowns" -Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force -Copy-Item "$($RepoRoot)/CONTRIBUTING.md" -Destination "${DocOutDir}/api/CONTRIBUTING.md" -Force - -Write-Verbose "Building site..." -& "${DocFxTool}" build "${DocOutDir}/docfx.json" - -Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force -Copy-Item "${DocGenDir}/assets/toc.yml" -Destination "${DocOutDir}/_site/" -Force \ No newline at end of file +if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $GetGithubIoDocIndexFn }).Count -gt 0) +{ + &$GetGithubIoDocIndexFn +} +else +{ + LogWarning "The function '$GetGithubIoDocIndexFn' was not found." +} \ No newline at end of file diff --git a/eng/pipelines/doc-index.yml b/eng/pipelines/doc-index.yml index 5cb453e6c..bd922a1f1 100644 --- a/eng/pipelines/doc-index.yml +++ b/eng/pipelines/doc-index.yml @@ -8,25 +8,23 @@ jobs: inputs: versionSpec: '3.6' - - template: /eng/common/pipelines/templates/steps/replace-relative-links.yml - parameters: - TargetFolder: '.' - RootFolder: '.' - BuildSHA: $(Build.SourceVersion) - RepoId: 'Azure/azure-sdk-for-cpp' - - pwsh: | Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://github.com/dotnet/docfx/releases/download/v2.43.2/docfx.zip" ` -OutFile "docfx.zip" | Wait-Process; Expand-Archive -Path "docfx.zip" -DestinationPath "./docfx/" + echo "##vso[task.setvariable variable=docfxPath;isOutput=true]$(Build.SourcesDirectory)/docfx/docfx.exe" workingDirectory: $(Build.SourcesDirectory) displayName: Download and Extract DocFX - - - pwsh: >- - $(Build.SourcesDirectory)/eng/docs/index/Generate-DocIndex.ps1 - -RepoRoot $(Build.SourcesDirectory) - -DocGenDir "$(Build.SourcesDirectory)/eng/docs/index" - -verbose - displayName: 'Generate Doc Index' + name: setupDocfxTool + - task: PowerShell@2 + displayName: "Generate Doc Index" + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/docs/index/Generate-DocIndex.ps1 + arguments: > + -Docfx $(setupDocfxTool.docfxPath) + -RepoRoot $(Build.SourcesDirectory) + -DocGenDir "$(Build.SourcesDirectory)/eng/docs/index" + -verbose - task: UsePythonVersion@0 displayName: 'Use Python 3.6' diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 7cc247ee9..5113c9e89 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -1,8 +1,8 @@ $Language = "cpp" $PackageRepository = "CPP" $packagePattern = "*.json" -$MetadataUri = "" - +$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/cpp-packages.csv" +$BlobStorageUrl = "https://azuresdkdocs.blob.core.windows.net/%24web?restype=container&comp=list&prefix=cpp%2F&delimiter=%2F" # Parse out package publishing information given a nupkg ZIP format. function Get-cpp-PackageInfoFromPackageFile($pkg, $workingDirectory) @@ -45,4 +45,15 @@ function Publish-cpp-GithubIODocs ($DocLocation, $PublicArtifactLocation) $packageInfo = (Get-Content (Join-Path $DocLocation 'package-info.json') | ConvertFrom-Json) $releaseTag = RetrieveReleaseTag "CPP" $PublicArtifactLocation Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version -ReleaseTag $releaseTag +} + +function Get-cpp-GithubIoDocIndex() { + # Fetch out all package metadata from csv file. + $metadata = Get-CSVMetadata -MetadataUri $MetadataUri + # Get the artifacts name from blob storage + $artifacts = Get-BlobStorage-Artifacts -blobStorageUrl $BlobStorageUrl -blobDirectoryRegex "^cpp/(.*)/$" -blobArtifactsReplacement '$1' + # Build up the artifact to service name mapping for GithubIo toc. + $tocContent = Get-TocMapping -metadata $metadata -artifacts $artifacts + # Generate yml/md toc files and build site. + GenerateDocfxTocContent -tocContent $tocContent -lang "C++" } \ No newline at end of file