Engineering changes required to publish tags and docs (#260)

* update version to dev.1 for test release

* Update release date in CHANGELOG.md

* Add checkout step

* Faster matrix

* Add Skip.Test to improve inner loop

* Show more info for tagging

* Revert "Faster matrix"

This reverts commit 713afd3d64743cbc1841e1ee965e936c049f7cf7.

* Add correct folder

* Add fixes for tagging repo

* Use Invoke-WebRequest for GetExistingTags

* Use -Uri

* return empty list

* Add vcpkg caching

* Set contents to copy

* Add doc uploads for C++

* Revert "Update release date in CHANGELOG.md"

This reverts commit 6fb302701381758cbb3d3828c2ce8d29d6b77950.

* Revert "update version to dev.1 for test release"

This reverts commit 4e188e4da99cb1246f195a5362e47d6ff80c3219.

* Revert "Add vcpkg caching"

This reverts commit 1f5f4d95e21d07d5ade1df470f39ceb478849f06.

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -347,6 +347,38 @@ function ParseCArtifact($pkg, $workingDirectory) {
   }
 }
 
+function ParseCppArtifact($pkg, $workingDirectory) {
+  $packageInfo = Get-Content -Raw -Path $pkg | ConvertFrom-JSON
+  $packageArtifactLocation = (Get-ItemProperty $pkg).Directory.FullName
+  $releaseNotes = ""
+  $readmeContent = ""
+
+  $pkgVersion = $packageInfo.version
+  $pkgName = $packageInfo.name
+
+  $changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0]
+  if ($changeLogLoc)
+  {
+    $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
+  }
+
+  $readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0]
+  if ($readmeContentLoc) {
+    $readmeContent = Get-Content -Raw $readmeContentLoc
+  }
+
+  return New-Object PSObject -Property @{
+    PackageId      = $pkgName
+    PackageVersion = $pkgVersion
+    # Artifact info is always considered deployable for now becasue it is not
+    # deployed anywhere. Dealing with duplicate tags happens downstream in
+    # CheckArtifactShaAgainstTagsList
+    Deployable     = $true
+    ReleaseNotes   = $releaseNotes
+  }
+}
+
+
 # Returns the pypi publish status of a package id and version.
 function IsPythonPackageVersionPublished($pkgId, $pkgVersion) {
   try {
@@ -374,9 +406,10 @@ function IsPythonPackageVersionPublished($pkgId, $pkgVersion) {
 # Retrieves the list of all tags that exist on the target repository
 function GetExistingTags($apiUrl) {
   try {
-    return (Invoke-WebRequest-WithHandling -Method "GET" -url "$apiUrl/git/refs/tags"  ) | % { $_.ref.Replace("refs/tags/", "") }
+    return (Invoke-WebRequest -Method "GET" -Uri "$apiUrl/git/refs/tags"  -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") }
   }
   catch {
+    Write-Host $_
     $statusCode = $_.Exception.Response.StatusCode.value__
     $statusDescription = $_.Exception.Response.StatusDescription
 
@@ -386,7 +419,7 @@ function GetExistingTags($apiUrl) {
 
     # Return an empty list if there are no tags in the repo
     if ($statusCode -eq 404) {
-      return @()
+      return ,@()
     }
 
     exit(1)
@@ -424,6 +457,10 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
       $ParsePkgInfoFn = "ParseCArtifact"
       $packagePattern = "*.json"
     }
+    "CPP" {
+      $ParsePkgInfoFn = "ParseCppArtifact"
+      $packagePattern = "*.json"
+    }
     default {
       Write-Host "Unrecognized Language: $language"
       exit(1)

eng/common/scripts/copy-docs-to-blobstorage.ps1

@@ -350,4 +350,10 @@ if ($Language -eq "c")
     # used to publish multiple docs packages in a single invocation.
     $pkgInfo = Get-Content $DocLocation/package-info.json | ConvertFrom-Json
     Upload-Blobs -DocDir $DocLocation -PkgName 'docs' -DocVersion $pkgInfo.version
+}
+
+if ($Language -eq "cpp")
+{
+    $packageInfo = (Get-Content (Join-Path $DocLocation 'package-info.json') | ConvertFrom-Json)
+    Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version
 }

eng/common/scripts/create-tags-and-git-release.ps1

@@ -7,7 +7,7 @@ param (
   $artifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory)
   $workingDirectory, # directory that package artifacts will be extracted into for examination (if necessary)
   $packageRepository, # used to indicate destination against which we will check the existing version.
-  # valid options: PyPI, Nuget, NPM, Maven, C
+  # valid options: PyPI, Nuget, NPM, Maven, C, CPP
   # used by CreateTags
   $releaseSha, # the SHA for the artifacts. DevOps: $(Release.Artifacts.<artifactAlias>.SourceVersion) or $(Build.SourceVersion)
 

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -6,6 +6,7 @@ parameters:
 
 jobs:
 - job: Validate
+  condition: and(succeededOrFailed(), ne(variables['Skip.Test'], 'true'))
   strategy:
     matrix:
       Linux_x64:
@@ -161,7 +162,8 @@ jobs:
 
           - task: CopyFiles@2
             inputs:
-              sourceFolder: $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/package-info.json
+              sourceFolder: $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}
+              contents: package-info.json
               targetFolder: build/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/docs/html
             displayName: Copy package-info.json to documentation path
 

eng/pipelines/templates/stages/archetype-cpp-release.yml

@@ -23,19 +23,16 @@ stages:
               runOnce:
                 deploy:
                   steps:
-                    - checkout: none
+                    - checkout: self
                     - pwsh: Get-ChildItem -Recurse $(Pipeline.Workspace)/packages/${{artifact.Name}}
                       displayName: Output Visible Artifacts
-                    - pwsh: >-
-                          $(Build.SourcesDirectory)/eng/common/create-tags-and-git-release.ps1
-                          -artifactLocation "$(Pipeline.Workspace)/packages/${{artifact.Name}}"
-                          -packageRepository C
-                          -releaseSha $(Build.SourceVersion)
-                          -repoId $(Build.Repository.Name)
-                      displayName: 'Verify Package Tags and Create Git Releases'
-                      timeoutInMinutes: 5
-                      env:
-                        GH_TOKEN: $(azuresdk-github-pat)
+
+                    - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
+                      parameters:
+                        ArtifactLocation: "$(Pipeline.Workspace)/packages/${{artifact.Name}}"
+                        PackageRepository: CPP
+                        ReleaseSha: $(Build.SourceVersion)
+                        RepoId: Azure/azure-sdk-for-cpp
 
           - ${{if ne(artifact.options.skipPublishDocs, 'true')}}:
             - deployment: PublishDocs
@@ -50,7 +47,7 @@ stages:
                 runOnce:
                   deploy:
                     steps:
-                      - checkout: none
+                      - checkout: self
                       - pwsh: Write-Host 'publish docs'
                       - pwsh: |
                           Get-ChildItem -Recurse $(Pipeline.Workspace)/docs/${{ artifact.Name }}