learn.microsoft.com docs onboarding engsys (#6818)

* Add metadata fetching

* Update docindex.yml to use new params and variables

* Add onboarding logic and update .gitignore to allow eng/scripts/docs edits

* Test: point to different branch

* Skip onboarding legacy moniker

* Add docs to release process

* REVERT: Test eng/common changes to Update-DocsMsPackages.ps1

* Remove nightly branch work, those require hosted docs

* Make CHANGELOG.md comply with rules

* Fix relevant spelling issues

* TEST: Add GitHub token

* Branch

* push branch

* Set DefinitionId

* Test branch

* Token

* Remove unnecessary condition

* Commitish

* Specify monikers

* Revert test-related changes

.gitignore

@@ -351,6 +351,7 @@ build/
 
 #Doxygen
 [Dd]ocs/
+!eng/scripts/docs
 
 # vscode
 **/.vscode/*

.vscode/cspell.json

@@ -310,6 +310,19 @@
     "zulu"
   ],
   "overrides": [
+    { 
+      "filename": "**/eng/pipelines/docindex.yml",
+      "words": [
+        "msdocs",
+        "SDKCI"
+      ]
+    },
+    { 
+      "filename": "**/eng/pipelines/templates/stages/archetype-cpp-release.yml",
+      "words": [
+        "docsms"
+      ]
+    },
     {
       "filename": "CMakePresets.json",
       "words": [

eng/pipelines/docindex.yml

@@ -1,4 +1,99 @@
 trigger: none
 
+parameters:
+- name: StartMainSDKCIRun
+  displayName: |
+    Kick off the main SDK CI docs run when manually running the pipeline
+  type: boolean
+  default: false
+
+variables:
+  - template: /eng/pipelines/templates/variables/globals.yml
+  - template: /eng/pipelines/templates/variables/image.yml
 jobs:
   - template: /eng/common/pipelines/templates/jobs/docindex.yml
+
+  - job: UpdateDocsMsBuildConfig
+    pool:
+      name: $(LINUXPOOL)
+      demands: ImageOverride -equals $(LINUXVMIMAGE)
+    variables:
+      DocRepoLocation: $(Pipeline.Workspace)/docs
+      DocRepoOwner: MicrosoftDocs
+      DocRepoName: azure-docs-sdk-cpp
+
+    steps:
+      # Sync docs repo (this can be sparse)
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+        parameters:
+          SkipCheckoutNone: true
+          Paths:
+            - ci-configs/
+            - metadata/
+            - docs-ref-mapping/
+            - docs-ref-services/
+          Repositories:
+            - Name: $(DocRepoOwner)/$(DocRepoName)
+              WorkingDirectory: $(DocRepoLocation)
+
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsPackages.ps1
+          arguments: -DocRepoLocation $(DocRepoLocation) -SkipLegacyOnboarding
+        displayName: Update Docs Onboarding for main branch
+
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1
+          arguments: >- 
+            -DocRepoLocation $(DocRepoLocation)
+            -Monikers @('latest','preview')
+        displayName: Generate Service Level Readme for main branch
+
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsToc.ps1
+          arguments: >-
+            -DocRepoLocation $(DocRepoLocation)
+            -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml
+        displayName: Generate ToC for main branch
+
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1
+          arguments: >-
+            -DocRepoLocation $(DocRepoLocation)
+        displayName: Verify Required Docs Json Members
+
+      # Push changes to docs repo
+      - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+        parameters:
+          WorkingDirectory: $(DocRepoLocation)
+
+      - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
+        parameters:
+          BaseRepoBranch: $(DefaultBranch)
+          BaseRepoOwner: $(DocRepoOwner)
+          CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)"
+          TargetRepoName: $(DocRepoName)
+          TargetRepoOwner: $(DocRepoOwner)
+          WorkingDirectory: $(DocRepoLocation)
+
+      - task: AzureCLI@2
+        displayName: Queue Docs CI build
+        condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(${{ parameters.StartMainSDKCIRun }}, true)))
+        inputs:
+          azureSubscription: msdocs-apidrop-connection
+          scriptType: pscore
+          scriptLocation: inlineScript
+          inlineScript: |
+            $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv
+            eng/common/scripts/Queue-Pipeline.ps1 `
+              -Organization "apidrop" `
+              -Project "Content%20CI" `
+              -DefinitionId 8056 `
+              -BearerToken $accessToken

eng/pipelines/templates/stages/archetype-cpp-release.yml

@@ -8,6 +8,9 @@ parameters:
   IntegrationDependsOn:
     - PrePublishBuild
 
+  TargetDocRepoOwner: MicrosoftDocs
+  TargetDocRepoName: azure-docs-sdk-cpp
+
 stages:
   - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}:
     - ${{ each artifact in parameters.Artifacts }}:
@@ -116,6 +119,40 @@ stages:
                     TargetLanguage: 'cpp'
                     ArtifactLocation: '$(Pipeline.Workspace)/packages/${{artifact.Name}}'
             
+            - job: PublishDocsMs
+              displayName: Docs.MS Release
+              condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true'))
+
+              # C++ learn docs are generated from hosted doxygen docs so this
+              # can only run after the static docs have been published.
+              dependsOn: PublishDocs
+
+              pool:
+                name: $(LINUXPOOL)
+                image: $(LINUXVMIMAGE)
+                os: linux
+              steps:
+                - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+                  parameters:
+                    Paths:
+                      - sdk/**/*.md
+                      - .github/CODEOWNERS
+                - download: current
+                - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
+                  parameters:
+                    PackageInfoLocations:
+                      - $(Pipeline.Workspace)/packages/PackageInfo/${{artifact.name}}.json
+                    RepoId: Azure/azure-sdk-for-cpp
+                    WorkingDirectory: $(System.DefaultWorkingDirectory)
+                    TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}}
+                    TargetDocRepoName: ${{parameters.TargetDocRepoName}}
+                    Language: cpp
+                    SparseCheckoutPaths:
+                      - docs-ref-services/
+                      - metadata/
+                      - ci-configs/packages-latest.json
+                      - ci-configs/packages-preview.json
+
           - ${{ if ne(artifact.skipPublishVcpkg, 'true') }}:
             - job: PublishVcpkg
               displayName: Publish to vcpkg

eng/scripts/Language-Settings.ps1

@@ -3,6 +3,10 @@ $LanguageDisplayName = "C++"
 $PackageRepository = "CPP"
 $packagePattern = "package-info.json"
 $MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/cpp-packages.csv"
+$GithubUri = "https://github.com/Azure/azure-sdk-for-cpp"
+$PackageRepositoryUri = "https://github.com/microsoft/vcpkg/tree/master/ports"
+
+. "$PSScriptRoot/docs/Docs-ToC.ps1"
 
 $VersionRegex = '(#define AZURE_\w+_VERSION_MAJOR )(?<major>[0-9]+)(\s+#define AZURE_\w+_VERSION_MINOR )(?<minor>[0-9]+)(\s+#define AZURE_\w+_VERSION_PATCH )(?<patch>[0-9]+)(\s+#define AZURE_\w+_VERSION_PRERELEASE )"(?<prerelease>[a-zA-Z0-9.]*)"';
 function Get-VersionHppLocation ($ServiceDirectory, $PackageName) {
@@ -11,6 +15,30 @@ function Get-VersionHppLocation ($ServiceDirectory, $PackageName) {
     return $versionHppLocation
 }
 
+function Get-cpp-DocsMsMetadataForPackage($PackageInfo) {
+  $readmeName = $PackageInfo.Name.ToLower()
+
+  # Readme names (which are used in the URL) should not include redundant terms
+  # when viewed in URL form. For example:
+  # https://docs.microsoft.com/en-us/dotnet/api/overview/azure/storage.blobs-readme
+  # Note how the end of the URL doesn't look like:
+  # ".../azure/azure.storage.blobs-readme"
+
+  # This logic eliminates a preceding "azure-" in the readme filename.
+  # "azure-storage-blobs" -> "storage-blobs"
+  if ($readmeName.StartsWith('azure-')) {
+    $readmeName = $readmeName.Substring(6)
+  }
+
+  Write-Host "Docs.ms Readme name: $($readmeName)"
+  New-Object PSObject -Property @{
+    DocsMsReadMeName      = $readmeName
+    LatestReadMeLocation  = 'docs-ref-services/latest'
+    PreviewReadMeLocation = 'docs-ref-services/preview'
+    Suffix                = ''
+  }
+}
+
 function Get-cpp-PackageInfoFromRepo($pkgPath, $serviceDirectory)
 {
   $pkgName = Split-Path -Leaf $pkgPath

eng/scripts/docs/Docs-Onboarding.ps1

@@ -0,0 +1,60 @@
+. "$PSScriptRoot/Docs-ToC.ps1"
+
+# $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding"
+function Set-cpp-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation, $packageSourceOverride) {
+  $onboardingFile = GetOnboardingFile `
+    -docRepoLocation $docRepoLocation `
+    -moniker $moniker
+
+  $onboardingSpec = Get-Content $onboardingFile -Raw | ConvertFrom-Json -AsHashtable
+
+  $packagesToOnboard = @()
+  foreach ($package in $metadata) { 
+    $packageSpec = [ordered]@{
+      name = $package.Name
+      tool = 'cpp-doxygen-1.9.1'
+      url = "https://azuresdkdocs.z19.web.core.windows.net/cpp/$($package.Name)/$($package.Version)/annotated.html"
+    }
+
+    $packagesToOnboard += $packageSpec
+  }
+
+  $onboardingSpec['packages'] = $packagesToOnboard
+
+  Set-Content `
+    -Path $onboardingFile `
+    -Value ($onboardingSpec | ConvertTo-Json -Depth 100)
+}
+
+# $GetDocsPackagesAlreadyOnboarded = "Get-${Language}-DocsPackagesAlreadyOnboarded"
+function Get-cpp-DocsPackagesAlreadyOnboarded($docRepoLocation, $moniker) {
+  $packageOnboardingFile = GetOnboardingFile `
+    -docRepoLocation $DocRepoLocation `
+    -moniker $moniker
+  
+  $onboardedPackages = @{}
+  $onboardingSpec = ConvertFrom-Json (Get-Content $packageOnboardingFile -Raw)
+  foreach ($spec in $onboardingSpec.packages) {
+    $packageInfo = GetPackageInfoFromDocsMsConfig $spec
+    $onboardedPackages[$packageInfo.Name] = $packageInfo
+  }
+
+  return $onboardedPackages
+}
+
+function GetPackageInfoFromDocsMsConfig($spec) {
+  if (!$spec) { 
+    throw "Spec must not be empty"
+  }
+
+  # "https://azuresdkdocs.z19.web.core.windows.net/cpp/azure-security-attestation/1.1.0/annotated.html" => 1.1.0
+  $version = $spec.url.Split('/')[-2]
+  if (!$version) {
+    throw "Could not determine version from URL: $($spec.url)"
+  }
+
+  return @{
+    Name    = $spec.name
+    Version = $version
+  }
+}

eng/scripts/docs/Docs-ToC.ps1

@@ -0,0 +1,85 @@
+function GetOnboardingFile($docRepoLocation, $moniker) { 
+  $packageOnboardingFile = "$docRepoLocation/ci-configs/packages-latest.json"
+  if ("preview" -eq $moniker) {
+    $packageOnboardingFile = "$docRepoLocation/ci-configs/packages-preview.json"
+  }
+
+  return $packageOnboardingFile
+}
+
+# $GetOnboardedDocsMsPackagesFn = "Get-${Language}-OnboardedDocsMsPackages"
+function Get-cpp-OnboardedDocsMsPackages($DocRepoLocation) {
+  $packageOnboardingFiles = @(
+    "$DocRepoLocation/ci-configs/packages-latest.json",
+    "$DocRepoLocation/ci-configs/packages-preview.json")
+
+  $onboardedPackages = @{}
+  foreach ($file in $packageOnboardingFiles) {
+    $onboardingSpec = ConvertFrom-Json (Get-Content $file -Raw)
+    foreach ($spec in $onboardingSpec.packages) {
+      $packageName = $spec.name
+      $onboardedPackages[$packageName] = $null
+    }
+  }
+
+  return $onboardedPackages
+}
+
+# $GetOnboardedDocsMsPackagesForMonikerFn = "Get-${Language}-OnboardedDocsMsPackagesForMoniker"
+function Get-cpp-OnboardedDocsMsPackagesForMoniker($DocRepoLocation, $moniker) {
+  $packageOnboardingFile = GetOnboardingFile `
+    -docRepoLocation $DocRepoLocation `
+    -moniker $moniker
+  
+  $onboardedPackages = @{}
+  $onboardingSpec = ConvertFrom-Json (Get-Content $packageOnboardingFile -Raw)
+  foreach ($spec in $onboardingSpec.packages) {
+    $jsonFile = "$DocRepoLocation/metadata/$($moniker)/$($spec.name).json"
+    if (Test-Path $jsonFile) {
+      $onboardedPackages[$spec.name] = ConvertFrom-Json (Get-Content $jsonFile -Raw)
+    }
+    else {
+      $onboardedPackages[$spec.name] = $null
+    }
+  }
+
+  return $onboardedPackages
+}
+
+# $GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme"
+function Get-cpp-PackageLevelReadme($packageMetadata) {
+  return GetPackageReadmeName -packageMetadata $packageMetadata
+}
+
+function GetPackageReadmeName($packageMetadata) {
+    # Fallback to get package-level readme name if metadata file info does not exist
+    $packageLevelReadmeName = $packageMetadata.Package.ToLower().Replace('azure-', '')
+  
+    # If there is a metadata json for the package use the DocsMsReadmeName from
+    # the metadata function
+    if ($packageMetadata.PSObject.Members.Name -contains "FileMetadata") {
+      $readmeMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageMetadata.FileMetadata
+      $packageLevelReadmeName = $readmeMetadata.DocsMsReadMeName
+    }
+    return $packageLevelReadmeName
+}
+
+# $GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
+function Get-cpp-DocsMsTocData($packageMetadata, $docRepoLocation) {
+  $packageLevelReadmeName = GetPackageReadmeName -packageMetadata $packageMetadata
+  $packageTocHeader = GetDocsTocDisplayName $packageMetadata
+  $output = [PSCustomObject]@{
+    PackageLevelReadmeHref = "~/docs-ref-services/{moniker}/$packageLevelReadmeName-readme.md"
+    PackageTocHeader       = $packageTocHeader
+    TocChildren            = @($packageMetadata.Package)
+  }
+
+  return $output
+}
+
+# $GetRepositoryLinkFn = "Get-${Language}-RepositoryLink"
+function Get-cpp-RepositoryLink($packageInfo) {
+  return "$PackageRepositoryUri/$($packageInfo.Package)-cpp"
+}
+
+

sdk/template/azure-template/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## 1.0.0 (Unreleased)
 
+### Other Changes
+
 * Validate template project ahead of release
 
 ## 1.0.0-beta.25 (2021-01-12)