Vcpkg publish framework (#645)

* Add vcpkg artifact creation and port template copying
* Skip staging if vcpkg port directory doesn't exist

eng/common/scripts/New-ReleaseAsset.ps1

@@ -0,0 +1,62 @@
+<#
+.SYNOPSIS
+Uploads the release asset and returns the resulting object from the upload
+
+.PARAMETER ReleaseTag
+Tag to look up release
+
+.PARAMETER AssetPath
+Location of the asset file to upload
+
+.PARAMETER GitHubRepo
+Name of the GitHub repo to search (of the form Azure/azure-sdk-for-cpp)
+
+#>
+
+param (
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $ReleaseTag,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $AssetPath,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $GitHubRepo,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $GitHubPat
+)
+
+# Get information about release at $ReleaseTag
+$releaseInfoUrl = "https://api.github.com/repos/$GitHubRepo/releases/tags/$ReleaseTag"
+Write-Verbose "Requesting release info from $releaseInfoUrl"
+$release = Invoke-RestMethod `
+    -Uri  $releaseInfoUrl `
+    -Method GET
+
+$assetFilename = Split-Path $AssetPath -Leaf
+
+# Upload URL comes in the literal form (yes, those curly braces) of:
+# https://uploads.github.com/repos/Azure/azure-sdk-for-cpp/releases/123/assets{?name,label}
+# Converts to something like:
+# https://uploads.github.com/repos/Azure/azure-sdk-for-cpp/releases/123/assets?name=foo.tar.gz
+# Docs: https://docs.github.com/en/rest/reference/repos#get-a-release-by-tag-name
+$uploadUrl = $release.upload_url.Split('{')[0] + "?name=$assetFilename"
+
+Write-Verbose "Uploading $assetFilename to $uploadUrl"
+
+$asset = Invoke-RestMethod `
+    -Uri $uploadUrl `
+    -Method POST `
+    -InFile $AssetPath `
+    -Credential $credentials `
+    -Headers @{ Authorization = "token $GitHubPat" } `
+    -ContentType "application/gzip"
+
+Write-Verbose "Upload complete. Browser download URL: $($asset.browser_download_url)"
+
+return $asset

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -148,7 +148,7 @@ jobs:
               $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }}/package-info.json
 
               $version = eng/scripts/Get-PkgVersion -ServiceDirectory ${{ parameters.ServiceDirectory }} -PackageName ${{ artifact.Path }}
-              $outputObject = @{ version = $version.ToString(); name = '${{ artifact.Name }}' } | ConvertTo-Json
+              $outputObject = @{ version = $version.ToString(); name = '${{ artifact.Name }}'; packageName = "${{ artifact.Name }}_$version"; } | ConvertTo-Json
 
               Set-Content -Path $outputPath -Value $outputObject
 
@@ -156,6 +156,41 @@ jobs:
               Write-Host "##vso[task.setvariable variable=AZURE_SDK_VERSION]$version"
             displayName: Create package info JSON file
 
+          # Building from the root of the repo at this point because of the
+          # nature of the CMake build system. The SoureDirectory should probably
+          # change.
+          - task: PowerShell@2
+            inputs:
+              targetType: filePath
+              filePath: eng/scripts/New-ReleaseArtifact.ps1
+              arguments: >-
+                -SourceDirectory ./
+                -PackageSpecPath $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/package-info.json
+                -Destination $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg
+                -Workspace $(Agent.TempDirectory)/vcpkg-${{ artifact.Name }}
+              pwsh: true
+            displayName: Compress publish artifact and stage for publishing
+
+          - task: Powershell@2
+            inputs:
+              targetType: inline
+              script: |
+                # If there is no vcpkg directory skip staging the vcpkg port artifacts
+                if (!(Test-Path sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Name }}/vcpkg/)) {
+                  exit
+                }
+
+                New-Item `
+                  -ItemType Directory `
+                  $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg/port -Force
+
+                Copy-Item `
+                  sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Name }}/vcpkg/* `
+                  $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg/port `
+                  -Exclude README.md
+              pwsh: true
+            displayName: Copy vcpkg port template files and remove .template extension
+
           - pwsh: |
               $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }}
               $changelogPath = "sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md"

eng/pipelines/templates/stages/archetype-cpp-release.yml

@@ -69,6 +69,70 @@ stages:
                           # we override the regular script path because we have cloned the build tools repo as a separate artifact.
                           ScriptPath: '$(Build.SourcesDirectory)/eng/common/scripts/copy-docs-to-blobstorage.ps1'
 
+          - ${{ if ne(artifact.skipPublishVcpkg, 'true') }}:
+            - deployment: PublishVcpkg
+              displayName: Publish to vcpkg
+              condition: ne(variables['Skip.PublishVcpkg'], 'true')
+              environment: github
+              # This step requires the tag step
+              dependsOn: TagRepository
+              pool:
+                vmImage: windows-2019
+
+              strategy:
+                runOnce:
+                  deploy:
+                    steps:
+                      - checkout: self
+
+                      - task: Powershell@2
+                        inputs:
+                          filePath: eng/scripts/Initialize-VcpkgRelease.ps1
+                          arguments: >-
+                            -GitHubPat $(azuresdk-github-pat)
+                            -GitHubRepo "$(Build.Repository.Name)"
+                            -PackageSpecPath $(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json
+                            -SourceDirectory $(Pipeline.Workspace)/packages/${{artifact.Name}}/vcpkg
+                            -Verbose
+                          pwsh: true
+                        displayName: Initialize vcpkg release
+
+                      - pwsh: |
+                          git clone https://github.com/microsoft/vcpkg $(Pipeline.Workspace)/vcpkg
+                          if ($LASTEXITCODE -ne 0) {
+                            Write-Error "Unable to check out vcpkg repo"
+                            exit $LASTEXITCODE
+                          }
+                          cd $(Pipeline.Workspace)/vcpkg
+
+                          # There is no need to check out a branch, the
+                          # create-pull-request.yml template creates the branch
+
+                          # Clean out the folder so that template files removed
+                          # are not inadvertently re-added
+                          if (Test-Path "ports/${{ artifact.Name }}") {
+                            Remove-Item -v -r "ports/${{ artifact.Name }}"
+                          }
+
+                          New-Item -Type Directory ports/${{ artifact.Name }}
+                          Copy-Item -Verbose $(Pipeline.Workspace)/packages/${{artifact.Name}}/vcpkg/port/* ports/${{ artifact.Name }}
+
+                          # Show artifacts copied into ports folder for PR
+                          Get-ChildItem -Recurse ports/${{ artifact.Name }}
+                        displayName: Copy updated files into vcpkg fork for PR
+                        condition: ne(variables['SkipVcpkgUpdate'], 'true')
+
+                      - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
+                        parameters:
+                          RepoOwner: Microsoft
+                          RepoName: vcpkg
+                          WorkingDirectory: $(Pipeline.Workspace)/vcpkg
+                          PrBranchName: azure-sdk-for-cpp-${{ artifact.Name }}-$(Build.BuildId)
+                          CommitMsg: "Update port for ${{ artifact.Name }}"
+                          PRTitle: "Update port for ${{ artifact.Name }}"
+                          BaseBranchName: master
+                          PushArgs: -f
+
           - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}:
             - deployment: UpdatePackageVersion
               displayName: "Update Package Version"

eng/scripts/Initialize-VcpkgRelease.ps1

@@ -0,0 +1,66 @@
+<#
+.SYNOPSIS
+Uploads the release asset and mutates files in $SourceDirectory/port to point to
+the uploaded GitHub release asset.
+
+.PARAMETER SourceDirectory
+Location of vcpkg assets (usually `<artifact-path>/packages/<package-name>/vcpkg`)
+
+.PARAMETER PackageSpecPath
+Location of the relevant package-info.json file
+
+.PARAMETER GitHubPat
+PAT for uploading asset to GitHub release and creating PR
+
+.PARAMETER GitHubRepo
+Name of the GitHub repo (of the form Azure/azure-sdk-for-cpp)
+
+#>
+
+param (
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $SourceDirectory,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $PackageSpecPath,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $GitHubPat,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $GitHubRepo
+)
+
+# If there's nothing in the "port" folder to upload set SkipVcpkgUpdate to true
+# and exit. Other steps will check SkipVcpkgUpdate to decide whether to move
+# forward.
+if (!(Get-ChildItem -Path "$SourceDirectory/port")) {
+    Write-Host "###vso[task.setvariable variable=SkipVcpkgUpdate]true"
+    exit
+}
+
+$packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json
+$assetFilename = "$($packageSpec.packageName).tar.gz"
+
+# Upload the asset to the release
+Write-Verbose "Uploading asset: $assetFilename..."
+$assetInfo = & $PSScriptRoot/../common/scripts/New-ReleaseAsset.ps1 `
+    -ReleaseTag $packageSpec.packageName `
+    -AssetPath $SourceDirectory/$assetFilename `
+    -GitHubRepo $GitHubRepo `
+    -GitHubPat $GitHubPat
+
+$sha512 = (Get-FileHash -Path "$SourceDirectory/$assetFilename" -Algorithm SHA512).Hash
+
+Write-Verbose "Mutating files with release info and creating PR"
+# Use asset URL to fill in vcpkg port tokens
+& $PSScriptRoot/New-VcpkgPortDefinition.ps1 `
+    -SourceDirectory "$SourceDirectory/port" `
+    -Version $packageSpec.Version `
+    -Url $assetInfo.browser_download_url `
+    -Filename $assetFilename `
+    -Sha512 $sha512

eng/scripts/New-ReleaseArtifact.ps1

@@ -0,0 +1,85 @@
+<#
+.SYNOPSIS
+Creates a .tar.gz archive of files in a given folder.
+
+.DESCRIPTION
+This is supposed to help zip up any location where artifacts are properly staged
+for placement in an archive. It does this by reading in the package information
+from the package.json file, then creating an archive named for the package and
+version, and copying the package to an artifact location.
+
+.PARAMETER SourceDirectory
+Source directory to compress. Files and folders in that source directory
+(except .git) will be compressed.
+
+.PARAMETER PackageSpecPath
+Location of the package.json file.
+
+.PARAMETER Destination
+Location to copy the artifact file to.
+
+.PARAMETER Destination
+Name of the destination to which the archive should be copied.
+
+.PARAMETER Workspace
+Workspace folder where assets are staged before creating.
+
+#>
+
+
+param (
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $SourceDirectory,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $PackageSpecPath,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $Destination,
+
+    [Parameter()]
+    [ValidateNotNullOrEmpty()]
+    [string] $Workspace = "$env:TEMP/vcpkg-workspace"
+)
+
+$initialLocation = Get-Location
+
+try {
+    $packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json
+    $archiveName = $packageSpec.packageName
+
+    Write-Verbose "Archive Name: $archiveName"
+
+    # Set up workspace, copy source to workspace
+    Remove-Item -Path $Workspace -Force -Recurse -ErrorAction Ignore
+    New-Item -ItemType directory -Path "$Workspace/$archiveName" -Force
+    Copy-Item `
+        -Path $SourceDirectory/* `
+        -Destination "$Workspace/$archiveName/" `
+        -Recurse `
+        -Exclude ".git"
+
+    # Move outside of workspace so the archive root contains a folder named
+    # after the package and its contents are the package contents. For example:
+    # azure-template-1.2.3/
+    #   CMakeLists.txt
+    #   ...
+    Set-Location $Workspace
+
+    # Create the tar.gz file
+    tar -cvz --exclude .git -f "$archiveName.tar.gz" $archiveName
+
+    New-Item -ItemType Directory $Destination -Force
+
+    # Copy release archive to the appropriate destination location
+    Copy-Item `
+    -Path "$archiveName.tar.gz" `
+    -Destination $Destination `
+    -Verbose
+
+} finally {
+    Set-Location $initialLocation
+}

eng/scripts/New-VcpkgPortDefinition.ps1

@@ -0,0 +1,61 @@
+<#
+.SYNOPSIS
+Replaces token content in each file of the SourceDirectory (assumes no subfolders)
+
+.PARAMETER SourceDirectory
+Location of vcpkg port files
+(usually `<artifact-path>/packages/<package-name>/vcpkg/port`)
+
+.PARAMETER Version
+Replaces %VERSION% token in files with the value of this parameter
+
+.PARAMETER Url
+Replaces %URL% token in files with the value of this parameter
+
+.PARAMETER Filename
+Replaces %FILENAME% token in files with the value of this parameter
+
+.PARAMETER Sha512
+Replaces %SHA512% token in files with the value of this parameter
+
+#>
+
+
+param (
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $SourceDirectory,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $Version,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $Url,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $Filename,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $Sha512
+)
+
+$files = Get-ChildItem -Path $SourceDirectory -Include *.template -Recurse
+
+Write-Host $files
+
+foreach ($file in $files) {
+    Write-Host "Updating file contents: $file"
+    $content = Get-Content -Raw -Path $file
+    $newContent = $content `
+        -replace '%VERSION%', $Version `
+        -replace '%URL%', $Url `
+        -replace '%FILENAME%', $Filename `
+        -replace '%SHA512%', $Sha512
+
+    $newContent | Set-Content $file
+    $file | Rename-Item -NewName { $_.Name -replace '.template', '' }
+}

sdk/template/azure-template/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Release History
 
+## 1.0.0-beta.7 (2020-09-15)
+
+* Validate vcpkg publishing automation framework (part 4)
+
+## 1.0.0-beta.6 (2020-09-10)
+
+* Validate vcpkg publishing automation framework (part 3)
+
+## 1.0.0-beta.5 (2020-09-10)
+
+* Validate vcpkg publishing automation framework (part 2)
+
+## 1.0.0-beta.4 (2020-09-10)
+
+* Validate vcpkg publishing automation framework
+
 ## 1.0.0-beta.2 (2020-09-09)
 
 * Validate engineering system changes

sdk/template/azure-template/inc/azure/template/version.hpp

@@ -8,7 +8,7 @@
 #define AZURE_TEMPLATE_VERSION_MAJOR 1
 #define AZURE_TEMPLATE_VERSION_MINOR 0
 #define AZURE_TEMPLATE_VERSION_PATCH 0
-#define AZURE_TEMPLATE_VERSION_PRERELEASE "beta.2"
+#define AZURE_TEMPLATE_VERSION_PRERELEASE "beta.7"
 
 namespace Azure { namespace Template {
 

sdk/template/azure-template/vcpkg/CONTROL.template

@@ -0,0 +1,4 @@
+Source: azure-template
+Version: %VERSION%
+Homepage: https://github.com/Azure/azure-sdk-for-cpp/tree/master/sdk/template/azure-template
+Description: Azure C++ SDK Template

sdk/template/azure-template/vcpkg/README.md

@@ -0,0 +1,4 @@
+# Vcpkg Publish Automation Assets
+
+These files are used by the engineering system to publish to vcpkg. They are not
+required to build the library.

sdk/template/azure-template/vcpkg/portfile.cmake.template

@@ -0,0 +1,80 @@
+# Common Ambient Variables:
+#   CURRENT_BUILDTREES_DIR    = ${VCPKG_ROOT_DIR}\buildtrees\${PORT}
+#   CURRENT_PACKAGES_DIR      = ${VCPKG_ROOT_DIR}\packages\${PORT}_${TARGET_TRIPLET}
+#   CURRENT_PORT_DIR          = ${VCPKG_ROOT_DIR}\ports\${PORT}
+#   CURRENT_INSTALLED_DIR     = ${VCPKG_ROOT_DIR}\installed\${TRIPLET}
+#   DOWNLOADS                 = ${VCPKG_ROOT_DIR}\downloads
+#   PORT                      = current port name (zlib, etc)
+#   TARGET_TRIPLET            = current triplet (x86-windows, x64-windows-static, etc)
+#   VCPKG_CRT_LINKAGE         = C runtime linkage type (static, dynamic)
+#   VCPKG_LIBRARY_LINKAGE     = target library linkage type (static, dynamic)
+#   VCPKG_ROOT_DIR            = <C:\path\to\current\vcpkg>
+#   VCPKG_TARGET_ARCHITECTURE = target architecture (x64, x86, arm)
+#   VCPKG_TOOLCHAIN           = ON OFF
+#   TRIPLET_SYSTEM_ARCH       = arm x86 x64
+#   BUILD_ARCH                = "Win32" "x64" "ARM"
+#   MSBUILD_PLATFORM          = "Win32"/"x64"/${TRIPLET_SYSTEM_ARCH}
+#   DEBUG_CONFIG              = "Debug Static" "Debug Dll"
+#   RELEASE_CONFIG            = "Release Static"" "Release DLL"
+#   VCPKG_TARGET_IS_WINDOWS
+#   VCPKG_TARGET_IS_UWP
+#   VCPKG_TARGET_IS_LINUX
+#   VCPKG_TARGET_IS_OSX
+#   VCPKG_TARGET_IS_FREEBSD
+#   VCPKG_TARGET_IS_ANDROID
+#   VCPKG_TARGET_IS_MINGW
+#   VCPKG_TARGET_EXECUTABLE_SUFFIX
+#   VCPKG_TARGET_STATIC_LIBRARY_SUFFIX
+#   VCPKG_TARGET_SHARED_LIBRARY_SUFFIX
+#
+# 	See additional helpful variables in /docs/maintainers/vcpkg_common_definitions.md
+
+# # Specifies if the port install should fail immediately given a condition
+# vcpkg_fail_port_install(MESSAGE "azure-template currently only supports Linux and Mac platforms" ON_TARGET "Windows")
+
+vcpkg_download_distfile(ARCHIVE
+    URLS "%URL%"
+    FILENAME "%FILENAME%"
+    SHA512 %SHA512%
+)
+
+vcpkg_extract_source_archive_ex(
+    OUT_SOURCE_PATH SOURCE_PATH
+    ARCHIVE ${ARCHIVE}
+    # (Optional) A friendly name to use instead of the filename of the archive (e.g.: a version number or tag).
+    # REF 1.0.0
+    # (Optional) Read the docs for how to generate patches at:
+    # https://github.com/Microsoft/vcpkg/blob/master/docs/examples/patching.md
+    # PATCHES
+    #   001_port_fixes.patch
+    #   002_more_port_fixes.patch
+)
+
+# # Check if one or more features are a part of a package installation.
+# # See /docs/maintainers/vcpkg_check_features.md for more details
+# vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS
+#   FEATURES # <- Keyword FEATURES is required because INVERTED_FEATURES are being used
+#     tbb   WITH_TBB
+#   INVERTED_FEATURES
+#     tbb   ROCKSDB_IGNORE_PACKAGE_TBB
+# )
+
+vcpkg_configure_cmake(
+    SOURCE_PATH ${SOURCE_PATH}
+    PREFER_NINJA # Disable this option if project cannot be built with Ninja
+    OPTIONS -DWARNINGS_AS_ERRORS=OFF
+    # OPTIONS_RELEASE -DOPTIMIZE=1
+    # OPTIONS_DEBUG -DDEBUGGABLE=1
+)
+
+vcpkg_install_cmake()
+
+# # Moves all .cmake files from /debug/share/azure-template/ to /share/azure-template/
+# # See /docs/maintainers/vcpkg_fixup_cmake_targets.md for more details
+# vcpkg_fixup_cmake_targets(CONFIG_PATH cmake TARGET_PATH share/azure-template)
+
+# # Handle copyright
+file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/azure-template RENAME copyright)
+
+# # Post-build test for cmake libraries
+# vcpkg_test_cmake(PACKAGE_NAME azure-template)