From 39e8dfd6e7c93c6777c47eb04777ca3ad2dcc51d Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 17 Sep 2020 15:45:43 -0700 Subject: [PATCH] Vcpkg publish framework (#645) * Add vcpkg artifact creation and port template copying * Skip staging if vcpkg port directory doesn't exist --- eng/common/scripts/New-ReleaseAsset.ps1 | 62 ++++++++++++++ .../templates/jobs/archetype-sdk-client.yml | 37 +++++++- .../stages/archetype-cpp-release.yml | 64 ++++++++++++++ eng/scripts/Initialize-VcpkgRelease.ps1 | 66 ++++++++++++++ eng/scripts/New-ReleaseArtifact.ps1 | 85 +++++++++++++++++++ eng/scripts/New-VcpkgPortDefinition.ps1 | 61 +++++++++++++ sdk/template/azure-template/CHANGELOG.md | 16 ++++ .../inc/azure/template/version.hpp | 2 +- .../azure-template/vcpkg/CONTROL.template | 4 + sdk/template/azure-template/vcpkg/README.md | 4 + .../vcpkg/portfile.cmake.template | 80 +++++++++++++++++ 11 files changed, 479 insertions(+), 2 deletions(-) create mode 100644 eng/common/scripts/New-ReleaseAsset.ps1 create mode 100644 eng/scripts/Initialize-VcpkgRelease.ps1 create mode 100644 eng/scripts/New-ReleaseArtifact.ps1 create mode 100644 eng/scripts/New-VcpkgPortDefinition.ps1 create mode 100644 sdk/template/azure-template/vcpkg/CONTROL.template create mode 100644 sdk/template/azure-template/vcpkg/README.md create mode 100644 sdk/template/azure-template/vcpkg/portfile.cmake.template diff --git a/eng/common/scripts/New-ReleaseAsset.ps1 b/eng/common/scripts/New-ReleaseAsset.ps1 new file mode 100644 index 000000000..83efcc097 --- /dev/null +++ b/eng/common/scripts/New-ReleaseAsset.ps1 @@ -0,0 +1,62 @@ +<# +.SYNOPSIS +Uploads the release asset and returns the resulting object from the upload + +.PARAMETER ReleaseTag +Tag to look up release + +.PARAMETER AssetPath +Location of the asset file to upload + +.PARAMETER GitHubRepo +Name of the GitHub repo to search (of the form Azure/azure-sdk-for-cpp) + +#> + +param ( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $ReleaseTag, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $AssetPath, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $GitHubRepo, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $GitHubPat +) + +# Get information about release at $ReleaseTag +$releaseInfoUrl = "https://api.github.com/repos/$GitHubRepo/releases/tags/$ReleaseTag" +Write-Verbose "Requesting release info from $releaseInfoUrl" +$release = Invoke-RestMethod ` + -Uri $releaseInfoUrl ` + -Method GET + +$assetFilename = Split-Path $AssetPath -Leaf + +# Upload URL comes in the literal form (yes, those curly braces) of: +# https://uploads.github.com/repos/Azure/azure-sdk-for-cpp/releases/123/assets{?name,label} +# Converts to something like: +# https://uploads.github.com/repos/Azure/azure-sdk-for-cpp/releases/123/assets?name=foo.tar.gz +# Docs: https://docs.github.com/en/rest/reference/repos#get-a-release-by-tag-name +$uploadUrl = $release.upload_url.Split('{')[0] + "?name=$assetFilename" + +Write-Verbose "Uploading $assetFilename to $uploadUrl" + +$asset = Invoke-RestMethod ` + -Uri $uploadUrl ` + -Method POST ` + -InFile $AssetPath ` + -Credential $credentials ` + -Headers @{ Authorization = "token $GitHubPat" } ` + -ContentType "application/gzip" + +Write-Verbose "Upload complete. Browser download URL: $($asset.browser_download_url)" + +return $asset diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index a2c3ab91a..6808596f6 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -148,7 +148,7 @@ jobs: $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }}/package-info.json $version = eng/scripts/Get-PkgVersion -ServiceDirectory ${{ parameters.ServiceDirectory }} -PackageName ${{ artifact.Path }} - $outputObject = @{ version = $version.ToString(); name = '${{ artifact.Name }}' } | ConvertTo-Json + $outputObject = @{ version = $version.ToString(); name = '${{ artifact.Name }}'; packageName = "${{ artifact.Name }}_$version"; } | ConvertTo-Json Set-Content -Path $outputPath -Value $outputObject @@ -156,6 +156,41 @@ jobs: Write-Host "##vso[task.setvariable variable=AZURE_SDK_VERSION]$version" displayName: Create package info JSON file + # Building from the root of the repo at this point because of the + # nature of the CMake build system. The SoureDirectory should probably + # change. + - task: PowerShell@2 + inputs: + targetType: filePath + filePath: eng/scripts/New-ReleaseArtifact.ps1 + arguments: >- + -SourceDirectory ./ + -PackageSpecPath $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/package-info.json + -Destination $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg + -Workspace $(Agent.TempDirectory)/vcpkg-${{ artifact.Name }} + pwsh: true + displayName: Compress publish artifact and stage for publishing + + - task: Powershell@2 + inputs: + targetType: inline + script: | + # If there is no vcpkg directory skip staging the vcpkg port artifacts + if (!(Test-Path sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Name }}/vcpkg/)) { + exit + } + + New-Item ` + -ItemType Directory ` + $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg/port -Force + + Copy-Item ` + sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Name }}/vcpkg/* ` + $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg/port ` + -Exclude README.md + pwsh: true + displayName: Copy vcpkg port template files and remove .template extension + - pwsh: | $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }} $changelogPath = "sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md" diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 9b2177679..25d534458 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -69,6 +69,70 @@ stages: # we override the regular script path because we have cloned the build tools repo as a separate artifact. ScriptPath: '$(Build.SourcesDirectory)/eng/common/scripts/copy-docs-to-blobstorage.ps1' + - ${{ if ne(artifact.skipPublishVcpkg, 'true') }}: + - deployment: PublishVcpkg + displayName: Publish to vcpkg + condition: ne(variables['Skip.PublishVcpkg'], 'true') + environment: github + # This step requires the tag step + dependsOn: TagRepository + pool: + vmImage: windows-2019 + + strategy: + runOnce: + deploy: + steps: + - checkout: self + + - task: Powershell@2 + inputs: + filePath: eng/scripts/Initialize-VcpkgRelease.ps1 + arguments: >- + -GitHubPat $(azuresdk-github-pat) + -GitHubRepo "$(Build.Repository.Name)" + -PackageSpecPath $(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json + -SourceDirectory $(Pipeline.Workspace)/packages/${{artifact.Name}}/vcpkg + -Verbose + pwsh: true + displayName: Initialize vcpkg release + + - pwsh: | + git clone https://github.com/microsoft/vcpkg $(Pipeline.Workspace)/vcpkg + if ($LASTEXITCODE -ne 0) { + Write-Error "Unable to check out vcpkg repo" + exit $LASTEXITCODE + } + cd $(Pipeline.Workspace)/vcpkg + + # There is no need to check out a branch, the + # create-pull-request.yml template creates the branch + + # Clean out the folder so that template files removed + # are not inadvertently re-added + if (Test-Path "ports/${{ artifact.Name }}") { + Remove-Item -v -r "ports/${{ artifact.Name }}" + } + + New-Item -Type Directory ports/${{ artifact.Name }} + Copy-Item -Verbose $(Pipeline.Workspace)/packages/${{artifact.Name}}/vcpkg/port/* ports/${{ artifact.Name }} + + # Show artifacts copied into ports folder for PR + Get-ChildItem -Recurse ports/${{ artifact.Name }} + displayName: Copy updated files into vcpkg fork for PR + condition: ne(variables['SkipVcpkgUpdate'], 'true') + + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoOwner: Microsoft + RepoName: vcpkg + WorkingDirectory: $(Pipeline.Workspace)/vcpkg + PrBranchName: azure-sdk-for-cpp-${{ artifact.Name }}-$(Build.BuildId) + CommitMsg: "Update port for ${{ artifact.Name }}" + PRTitle: "Update port for ${{ artifact.Name }}" + BaseBranchName: master + PushArgs: -f + - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}: - deployment: UpdatePackageVersion displayName: "Update Package Version" diff --git a/eng/scripts/Initialize-VcpkgRelease.ps1 b/eng/scripts/Initialize-VcpkgRelease.ps1 new file mode 100644 index 000000000..6e01f0443 --- /dev/null +++ b/eng/scripts/Initialize-VcpkgRelease.ps1 @@ -0,0 +1,66 @@ +<# +.SYNOPSIS +Uploads the release asset and mutates files in $SourceDirectory/port to point to +the uploaded GitHub release asset. + +.PARAMETER SourceDirectory +Location of vcpkg assets (usually `/packages//vcpkg`) + +.PARAMETER PackageSpecPath +Location of the relevant package-info.json file + +.PARAMETER GitHubPat +PAT for uploading asset to GitHub release and creating PR + +.PARAMETER GitHubRepo +Name of the GitHub repo (of the form Azure/azure-sdk-for-cpp) + +#> + +param ( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $SourceDirectory, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $PackageSpecPath, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $GitHubPat, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $GitHubRepo +) + +# If there's nothing in the "port" folder to upload set SkipVcpkgUpdate to true +# and exit. Other steps will check SkipVcpkgUpdate to decide whether to move +# forward. +if (!(Get-ChildItem -Path "$SourceDirectory/port")) { + Write-Host "###vso[task.setvariable variable=SkipVcpkgUpdate]true" + exit +} + +$packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json +$assetFilename = "$($packageSpec.packageName).tar.gz" + +# Upload the asset to the release +Write-Verbose "Uploading asset: $assetFilename..." +$assetInfo = & $PSScriptRoot/../common/scripts/New-ReleaseAsset.ps1 ` + -ReleaseTag $packageSpec.packageName ` + -AssetPath $SourceDirectory/$assetFilename ` + -GitHubRepo $GitHubRepo ` + -GitHubPat $GitHubPat + +$sha512 = (Get-FileHash -Path "$SourceDirectory/$assetFilename" -Algorithm SHA512).Hash + +Write-Verbose "Mutating files with release info and creating PR" +# Use asset URL to fill in vcpkg port tokens +& $PSScriptRoot/New-VcpkgPortDefinition.ps1 ` + -SourceDirectory "$SourceDirectory/port" ` + -Version $packageSpec.Version ` + -Url $assetInfo.browser_download_url ` + -Filename $assetFilename ` + -Sha512 $sha512 diff --git a/eng/scripts/New-ReleaseArtifact.ps1 b/eng/scripts/New-ReleaseArtifact.ps1 new file mode 100644 index 000000000..8ec174ce5 --- /dev/null +++ b/eng/scripts/New-ReleaseArtifact.ps1 @@ -0,0 +1,85 @@ +<# +.SYNOPSIS +Creates a .tar.gz archive of files in a given folder. + +.DESCRIPTION +This is supposed to help zip up any location where artifacts are properly staged +for placement in an archive. It does this by reading in the package information +from the package.json file, then creating an archive named for the package and +version, and copying the package to an artifact location. + +.PARAMETER SourceDirectory +Source directory to compress. Files and folders in that source directory +(except .git) will be compressed. + +.PARAMETER PackageSpecPath +Location of the package.json file. + +.PARAMETER Destination +Location to copy the artifact file to. + +.PARAMETER Destination +Name of the destination to which the archive should be copied. + +.PARAMETER Workspace +Workspace folder where assets are staged before creating. + +#> + + +param ( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $SourceDirectory, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $PackageSpecPath, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $Destination, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string] $Workspace = "$env:TEMP/vcpkg-workspace" +) + +$initialLocation = Get-Location + +try { + $packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json + $archiveName = $packageSpec.packageName + + Write-Verbose "Archive Name: $archiveName" + + # Set up workspace, copy source to workspace + Remove-Item -Path $Workspace -Force -Recurse -ErrorAction Ignore + New-Item -ItemType directory -Path "$Workspace/$archiveName" -Force + Copy-Item ` + -Path $SourceDirectory/* ` + -Destination "$Workspace/$archiveName/" ` + -Recurse ` + -Exclude ".git" + + # Move outside of workspace so the archive root contains a folder named + # after the package and its contents are the package contents. For example: + # azure-template-1.2.3/ + # CMakeLists.txt + # ... + Set-Location $Workspace + + # Create the tar.gz file + tar -cvz --exclude .git -f "$archiveName.tar.gz" $archiveName + + New-Item -ItemType Directory $Destination -Force + + # Copy release archive to the appropriate destination location + Copy-Item ` + -Path "$archiveName.tar.gz" ` + -Destination $Destination ` + -Verbose + +} finally { + Set-Location $initialLocation +} diff --git a/eng/scripts/New-VcpkgPortDefinition.ps1 b/eng/scripts/New-VcpkgPortDefinition.ps1 new file mode 100644 index 000000000..cdde8e81a --- /dev/null +++ b/eng/scripts/New-VcpkgPortDefinition.ps1 @@ -0,0 +1,61 @@ +<# +.SYNOPSIS +Replaces token content in each file of the SourceDirectory (assumes no subfolders) + +.PARAMETER SourceDirectory +Location of vcpkg port files +(usually `/packages//vcpkg/port`) + +.PARAMETER Version +Replaces %VERSION% token in files with the value of this parameter + +.PARAMETER Url +Replaces %URL% token in files with the value of this parameter + +.PARAMETER Filename +Replaces %FILENAME% token in files with the value of this parameter + +.PARAMETER Sha512 +Replaces %SHA512% token in files with the value of this parameter + +#> + + +param ( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $SourceDirectory, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $Version, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $Url, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $Filename, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $Sha512 +) + +$files = Get-ChildItem -Path $SourceDirectory -Include *.template -Recurse + +Write-Host $files + +foreach ($file in $files) { + Write-Host "Updating file contents: $file" + $content = Get-Content -Raw -Path $file + $newContent = $content ` + -replace '%VERSION%', $Version ` + -replace '%URL%', $Url ` + -replace '%FILENAME%', $Filename ` + -replace '%SHA512%', $Sha512 + + $newContent | Set-Content $file + $file | Rename-Item -NewName { $_.Name -replace '.template', '' } +} diff --git a/sdk/template/azure-template/CHANGELOG.md b/sdk/template/azure-template/CHANGELOG.md index 24187c4ba..f6bca1c56 100644 --- a/sdk/template/azure-template/CHANGELOG.md +++ b/sdk/template/azure-template/CHANGELOG.md @@ -1,5 +1,21 @@ # Release History +## 1.0.0-beta.7 (2020-09-15) + +* Validate vcpkg publishing automation framework (part 4) + +## 1.0.0-beta.6 (2020-09-10) + +* Validate vcpkg publishing automation framework (part 3) + +## 1.0.0-beta.5 (2020-09-10) + +* Validate vcpkg publishing automation framework (part 2) + +## 1.0.0-beta.4 (2020-09-10) + +* Validate vcpkg publishing automation framework + ## 1.0.0-beta.2 (2020-09-09) * Validate engineering system changes diff --git a/sdk/template/azure-template/inc/azure/template/version.hpp b/sdk/template/azure-template/inc/azure/template/version.hpp index 6517f1a45..e8182d5c9 100644 --- a/sdk/template/azure-template/inc/azure/template/version.hpp +++ b/sdk/template/azure-template/inc/azure/template/version.hpp @@ -8,7 +8,7 @@ #define AZURE_TEMPLATE_VERSION_MAJOR 1 #define AZURE_TEMPLATE_VERSION_MINOR 0 #define AZURE_TEMPLATE_VERSION_PATCH 0 -#define AZURE_TEMPLATE_VERSION_PRERELEASE "beta.2" +#define AZURE_TEMPLATE_VERSION_PRERELEASE "beta.7" namespace Azure { namespace Template { diff --git a/sdk/template/azure-template/vcpkg/CONTROL.template b/sdk/template/azure-template/vcpkg/CONTROL.template new file mode 100644 index 000000000..6efbbad1e --- /dev/null +++ b/sdk/template/azure-template/vcpkg/CONTROL.template @@ -0,0 +1,4 @@ +Source: azure-template +Version: %VERSION% +Homepage: https://github.com/Azure/azure-sdk-for-cpp/tree/master/sdk/template/azure-template +Description: Azure C++ SDK Template diff --git a/sdk/template/azure-template/vcpkg/README.md b/sdk/template/azure-template/vcpkg/README.md new file mode 100644 index 000000000..75c513be1 --- /dev/null +++ b/sdk/template/azure-template/vcpkg/README.md @@ -0,0 +1,4 @@ +# Vcpkg Publish Automation Assets + +These files are used by the engineering system to publish to vcpkg. They are not +required to build the library. \ No newline at end of file diff --git a/sdk/template/azure-template/vcpkg/portfile.cmake.template b/sdk/template/azure-template/vcpkg/portfile.cmake.template new file mode 100644 index 000000000..3564e626b --- /dev/null +++ b/sdk/template/azure-template/vcpkg/portfile.cmake.template @@ -0,0 +1,80 @@ +# Common Ambient Variables: +# CURRENT_BUILDTREES_DIR = ${VCPKG_ROOT_DIR}\buildtrees\${PORT} +# CURRENT_PACKAGES_DIR = ${VCPKG_ROOT_DIR}\packages\${PORT}_${TARGET_TRIPLET} +# CURRENT_PORT_DIR = ${VCPKG_ROOT_DIR}\ports\${PORT} +# CURRENT_INSTALLED_DIR = ${VCPKG_ROOT_DIR}\installed\${TRIPLET} +# DOWNLOADS = ${VCPKG_ROOT_DIR}\downloads +# PORT = current port name (zlib, etc) +# TARGET_TRIPLET = current triplet (x86-windows, x64-windows-static, etc) +# VCPKG_CRT_LINKAGE = C runtime linkage type (static, dynamic) +# VCPKG_LIBRARY_LINKAGE = target library linkage type (static, dynamic) +# VCPKG_ROOT_DIR = +# VCPKG_TARGET_ARCHITECTURE = target architecture (x64, x86, arm) +# VCPKG_TOOLCHAIN = ON OFF +# TRIPLET_SYSTEM_ARCH = arm x86 x64 +# BUILD_ARCH = "Win32" "x64" "ARM" +# MSBUILD_PLATFORM = "Win32"/"x64"/${TRIPLET_SYSTEM_ARCH} +# DEBUG_CONFIG = "Debug Static" "Debug Dll" +# RELEASE_CONFIG = "Release Static"" "Release DLL" +# VCPKG_TARGET_IS_WINDOWS +# VCPKG_TARGET_IS_UWP +# VCPKG_TARGET_IS_LINUX +# VCPKG_TARGET_IS_OSX +# VCPKG_TARGET_IS_FREEBSD +# VCPKG_TARGET_IS_ANDROID +# VCPKG_TARGET_IS_MINGW +# VCPKG_TARGET_EXECUTABLE_SUFFIX +# VCPKG_TARGET_STATIC_LIBRARY_SUFFIX +# VCPKG_TARGET_SHARED_LIBRARY_SUFFIX +# +# See additional helpful variables in /docs/maintainers/vcpkg_common_definitions.md + +# # Specifies if the port install should fail immediately given a condition +# vcpkg_fail_port_install(MESSAGE "azure-template currently only supports Linux and Mac platforms" ON_TARGET "Windows") + +vcpkg_download_distfile(ARCHIVE + URLS "%URL%" + FILENAME "%FILENAME%" + SHA512 %SHA512% +) + +vcpkg_extract_source_archive_ex( + OUT_SOURCE_PATH SOURCE_PATH + ARCHIVE ${ARCHIVE} + # (Optional) A friendly name to use instead of the filename of the archive (e.g.: a version number or tag). + # REF 1.0.0 + # (Optional) Read the docs for how to generate patches at: + # https://github.com/Microsoft/vcpkg/blob/master/docs/examples/patching.md + # PATCHES + # 001_port_fixes.patch + # 002_more_port_fixes.patch +) + +# # Check if one or more features are a part of a package installation. +# # See /docs/maintainers/vcpkg_check_features.md for more details +# vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS +# FEATURES # <- Keyword FEATURES is required because INVERTED_FEATURES are being used +# tbb WITH_TBB +# INVERTED_FEATURES +# tbb ROCKSDB_IGNORE_PACKAGE_TBB +# ) + +vcpkg_configure_cmake( + SOURCE_PATH ${SOURCE_PATH} + PREFER_NINJA # Disable this option if project cannot be built with Ninja + OPTIONS -DWARNINGS_AS_ERRORS=OFF + # OPTIONS_RELEASE -DOPTIMIZE=1 + # OPTIONS_DEBUG -DDEBUGGABLE=1 +) + +vcpkg_install_cmake() + +# # Moves all .cmake files from /debug/share/azure-template/ to /share/azure-template/ +# # See /docs/maintainers/vcpkg_fixup_cmake_targets.md for more details +# vcpkg_fixup_cmake_targets(CONFIG_PATH cmake TARGET_PATH share/azure-template) + +# # Handle copyright +file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/azure-template RENAME copyright) + +# # Post-build test for cmake libraries +# vcpkg_test_cmake(PACKAGE_NAME azure-template)