diff --git a/.gitignore b/.gitignore index e04060d3d..3d225926e 100644 --- a/.gitignore +++ b/.gitignore @@ -351,6 +351,7 @@ build/ #Doxygen [Dd]ocs/ +!eng/scripts/docs # vscode **/.vscode/* diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 843f91b8a..cab85438a 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -310,6 +310,19 @@ "zulu" ], "overrides": [ + { + "filename": "**/eng/pipelines/docindex.yml", + "words": [ + "msdocs", + "SDKCI" + ] + }, + { + "filename": "**/eng/pipelines/templates/stages/archetype-cpp-release.yml", + "words": [ + "docsms" + ] + }, { "filename": "CMakePresets.json", "words": [ diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 173eed02d..31c863c04 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -1,4 +1,99 @@ trigger: none +parameters: +- name: StartMainSDKCIRun + displayName: | + Kick off the main SDK CI docs run when manually running the pipeline + type: boolean + default: false + +variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml jobs: - template: /eng/common/pipelines/templates/jobs/docindex.yml + + - job: UpdateDocsMsBuildConfig + pool: + name: $(LINUXPOOL) + demands: ImageOverride -equals $(LINUXVMIMAGE) + variables: + DocRepoLocation: $(Pipeline.Workspace)/docs + DocRepoOwner: MicrosoftDocs + DocRepoName: azure-docs-sdk-cpp + + steps: + # Sync docs repo (this can be sparse) + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipCheckoutNone: true + Paths: + - ci-configs/ + - metadata/ + - docs-ref-mapping/ + - docs-ref-services/ + Repositories: + - Name: $(DocRepoOwner)/$(DocRepoName) + WorkingDirectory: $(DocRepoLocation) + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) -SkipLegacyOnboarding + displayName: Update Docs Onboarding for main branch + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -Monikers @('latest','preview') + displayName: Generate Service Level Readme for main branch + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsToc.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml + displayName: Generate ToC for main branch + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + displayName: Verify Required Docs Json Members + + # Push changes to docs repo + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + parameters: + WorkingDirectory: $(DocRepoLocation) + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DefaultBranch) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + + - task: AzureCLI@2 + displayName: Queue Docs CI build + condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(${{ parameters.StartMainSDKCIRun }}, true))) + inputs: + azureSubscription: msdocs-apidrop-connection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv + eng/common/scripts/Queue-Pipeline.ps1 ` + -Organization "apidrop" ` + -Project "Content%20CI" ` + -DefinitionId 8056 ` + -BearerToken $accessToken diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 7c2b339a3..4a299fc4a 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -8,6 +8,9 @@ parameters: IntegrationDependsOn: - PrePublishBuild + TargetDocRepoOwner: MicrosoftDocs + TargetDocRepoName: azure-docs-sdk-cpp + stages: - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - ${{ each artifact in parameters.Artifacts }}: @@ -115,6 +118,40 @@ stages: FolderForUpload: '$(Pipeline.Workspace)/docs/${{ artifact.Name }}' TargetLanguage: 'cpp' ArtifactLocation: '$(Pipeline.Workspace)/packages/${{artifact.Name}}' + + - job: PublishDocsMs + displayName: Docs.MS Release + condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) + + # C++ learn docs are generated from hosted doxygen docs so this + # can only run after the static docs have been published. + dependsOn: PublishDocs + + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - sdk/**/*.md + - .github/CODEOWNERS + - download: current + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - $(Pipeline.Workspace)/packages/PackageInfo/${{artifact.name}}.json + RepoId: Azure/azure-sdk-for-cpp + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: cpp + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ + - ci-configs/packages-latest.json + - ci-configs/packages-preview.json - ${{ if ne(artifact.skipPublishVcpkg, 'true') }}: - job: PublishVcpkg diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 738d7f2be..f51a3b0f1 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -3,6 +3,10 @@ $LanguageDisplayName = "C++" $PackageRepository = "CPP" $packagePattern = "package-info.json" $MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/cpp-packages.csv" +$GithubUri = "https://github.com/Azure/azure-sdk-for-cpp" +$PackageRepositoryUri = "https://github.com/microsoft/vcpkg/tree/master/ports" + +. "$PSScriptRoot/docs/Docs-ToC.ps1" $VersionRegex = '(#define AZURE_\w+_VERSION_MAJOR )(?[0-9]+)(\s+#define AZURE_\w+_VERSION_MINOR )(?[0-9]+)(\s+#define AZURE_\w+_VERSION_PATCH )(?[0-9]+)(\s+#define AZURE_\w+_VERSION_PRERELEASE )"(?[a-zA-Z0-9.]*)"'; function Get-VersionHppLocation ($ServiceDirectory, $PackageName) { @@ -11,6 +15,30 @@ function Get-VersionHppLocation ($ServiceDirectory, $PackageName) { return $versionHppLocation } +function Get-cpp-DocsMsMetadataForPackage($PackageInfo) { + $readmeName = $PackageInfo.Name.ToLower() + + # Readme names (which are used in the URL) should not include redundant terms + # when viewed in URL form. For example: + # https://docs.microsoft.com/en-us/dotnet/api/overview/azure/storage.blobs-readme + # Note how the end of the URL doesn't look like: + # ".../azure/azure.storage.blobs-readme" + + # This logic eliminates a preceding "azure-" in the readme filename. + # "azure-storage-blobs" -> "storage-blobs" + if ($readmeName.StartsWith('azure-')) { + $readmeName = $readmeName.Substring(6) + } + + Write-Host "Docs.ms Readme name: $($readmeName)" + New-Object PSObject -Property @{ + DocsMsReadMeName = $readmeName + LatestReadMeLocation = 'docs-ref-services/latest' + PreviewReadMeLocation = 'docs-ref-services/preview' + Suffix = '' + } +} + function Get-cpp-PackageInfoFromRepo($pkgPath, $serviceDirectory) { $pkgName = Split-Path -Leaf $pkgPath diff --git a/eng/scripts/docs/Docs-Onboarding.ps1 b/eng/scripts/docs/Docs-Onboarding.ps1 new file mode 100644 index 000000000..8d2c38aeb --- /dev/null +++ b/eng/scripts/docs/Docs-Onboarding.ps1 @@ -0,0 +1,60 @@ +. "$PSScriptRoot/Docs-ToC.ps1" + +# $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding" +function Set-cpp-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation, $packageSourceOverride) { + $onboardingFile = GetOnboardingFile ` + -docRepoLocation $docRepoLocation ` + -moniker $moniker + + $onboardingSpec = Get-Content $onboardingFile -Raw | ConvertFrom-Json -AsHashtable + + $packagesToOnboard = @() + foreach ($package in $metadata) { + $packageSpec = [ordered]@{ + name = $package.Name + tool = 'cpp-doxygen-1.9.1' + url = "https://azuresdkdocs.z19.web.core.windows.net/cpp/$($package.Name)/$($package.Version)/annotated.html" + } + + $packagesToOnboard += $packageSpec + } + + $onboardingSpec['packages'] = $packagesToOnboard + + Set-Content ` + -Path $onboardingFile ` + -Value ($onboardingSpec | ConvertTo-Json -Depth 100) +} + +# $GetDocsPackagesAlreadyOnboarded = "Get-${Language}-DocsPackagesAlreadyOnboarded" +function Get-cpp-DocsPackagesAlreadyOnboarded($docRepoLocation, $moniker) { + $packageOnboardingFile = GetOnboardingFile ` + -docRepoLocation $DocRepoLocation ` + -moniker $moniker + + $onboardedPackages = @{} + $onboardingSpec = ConvertFrom-Json (Get-Content $packageOnboardingFile -Raw) + foreach ($spec in $onboardingSpec.packages) { + $packageInfo = GetPackageInfoFromDocsMsConfig $spec + $onboardedPackages[$packageInfo.Name] = $packageInfo + } + + return $onboardedPackages +} + +function GetPackageInfoFromDocsMsConfig($spec) { + if (!$spec) { + throw "Spec must not be empty" + } + + # "https://azuresdkdocs.z19.web.core.windows.net/cpp/azure-security-attestation/1.1.0/annotated.html" => 1.1.0 + $version = $spec.url.Split('/')[-2] + if (!$version) { + throw "Could not determine version from URL: $($spec.url)" + } + + return @{ + Name = $spec.name + Version = $version + } +} diff --git a/eng/scripts/docs/Docs-ToC.ps1 b/eng/scripts/docs/Docs-ToC.ps1 new file mode 100644 index 000000000..5673d9127 --- /dev/null +++ b/eng/scripts/docs/Docs-ToC.ps1 @@ -0,0 +1,85 @@ +function GetOnboardingFile($docRepoLocation, $moniker) { + $packageOnboardingFile = "$docRepoLocation/ci-configs/packages-latest.json" + if ("preview" -eq $moniker) { + $packageOnboardingFile = "$docRepoLocation/ci-configs/packages-preview.json" + } + + return $packageOnboardingFile +} + +# $GetOnboardedDocsMsPackagesFn = "Get-${Language}-OnboardedDocsMsPackages" +function Get-cpp-OnboardedDocsMsPackages($DocRepoLocation) { + $packageOnboardingFiles = @( + "$DocRepoLocation/ci-configs/packages-latest.json", + "$DocRepoLocation/ci-configs/packages-preview.json") + + $onboardedPackages = @{} + foreach ($file in $packageOnboardingFiles) { + $onboardingSpec = ConvertFrom-Json (Get-Content $file -Raw) + foreach ($spec in $onboardingSpec.packages) { + $packageName = $spec.name + $onboardedPackages[$packageName] = $null + } + } + + return $onboardedPackages +} + +# $GetOnboardedDocsMsPackagesForMonikerFn = "Get-${Language}-OnboardedDocsMsPackagesForMoniker" +function Get-cpp-OnboardedDocsMsPackagesForMoniker($DocRepoLocation, $moniker) { + $packageOnboardingFile = GetOnboardingFile ` + -docRepoLocation $DocRepoLocation ` + -moniker $moniker + + $onboardedPackages = @{} + $onboardingSpec = ConvertFrom-Json (Get-Content $packageOnboardingFile -Raw) + foreach ($spec in $onboardingSpec.packages) { + $jsonFile = "$DocRepoLocation/metadata/$($moniker)/$($spec.name).json" + if (Test-Path $jsonFile) { + $onboardedPackages[$spec.name] = ConvertFrom-Json (Get-Content $jsonFile -Raw) + } + else { + $onboardedPackages[$spec.name] = $null + } + } + + return $onboardedPackages +} + +# $GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme" +function Get-cpp-PackageLevelReadme($packageMetadata) { + return GetPackageReadmeName -packageMetadata $packageMetadata +} + +function GetPackageReadmeName($packageMetadata) { + # Fallback to get package-level readme name if metadata file info does not exist + $packageLevelReadmeName = $packageMetadata.Package.ToLower().Replace('azure-', '') + + # If there is a metadata json for the package use the DocsMsReadmeName from + # the metadata function + if ($packageMetadata.PSObject.Members.Name -contains "FileMetadata") { + $readmeMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageMetadata.FileMetadata + $packageLevelReadmeName = $readmeMetadata.DocsMsReadMeName + } + return $packageLevelReadmeName +} + +# $GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData" +function Get-cpp-DocsMsTocData($packageMetadata, $docRepoLocation) { + $packageLevelReadmeName = GetPackageReadmeName -packageMetadata $packageMetadata + $packageTocHeader = GetDocsTocDisplayName $packageMetadata + $output = [PSCustomObject]@{ + PackageLevelReadmeHref = "~/docs-ref-services/{moniker}/$packageLevelReadmeName-readme.md" + PackageTocHeader = $packageTocHeader + TocChildren = @($packageMetadata.Package) + } + + return $output +} + +# $GetRepositoryLinkFn = "Get-${Language}-RepositoryLink" +function Get-cpp-RepositoryLink($packageInfo) { + return "$PackageRepositoryUri/$($packageInfo.Package)-cpp" +} + + diff --git a/sdk/template/azure-template/CHANGELOG.md b/sdk/template/azure-template/CHANGELOG.md index 134b0f1ac..f95e50fc8 100644 --- a/sdk/template/azure-template/CHANGELOG.md +++ b/sdk/template/azure-template/CHANGELOG.md @@ -2,6 +2,8 @@ ## 1.0.0 (Unreleased) +### Other Changes + * Validate template project ahead of release ## 1.0.0-beta.25 (2021-01-12)