Add nightly vcpkg publishing (#2025)

* Tool changes to support nightly test package release

* Initial wiring up and refactor vcpkg-publish.yml

* Use vcpkg-clone.yml

* Disable publishing for verification of nightly builds

* Complete the comment

* Dependency is enforced at the stage level, not the deployment/job level

* Remove environment, no approval needed

* Add ability to enable test release parameters

* deploy -> job

* deployment -> job

* Remove strategy

* Download pipeline artifacts

* task -> download

* Set appropriate working directory

* git status

* Set working directory

* Add pipeline for nightly vcpkg PR update

* Correct vcpkg-clone.yml path

* Add identity parameters to git merge commands

* Remove GitIdentityParameters, the merge command does not suppor tthem. Add instead in the pipeline

* Split lines

* Write-Host

* Template for nightly branch name, update comments, Check for scheduling or "PublishNightlyVcpkg"

* include archetype-cpp-release.yml changes

* Re-enable publishing stage

* Close the and

* Runtime condition for integration stage

* Move integration below package publishing stages

* try -- to disambiguate

* ^ -> ~

* Remove --

* Documentation and variable naming

* Move documentation comment

* Update eng/pipelines/templates/steps/generate-nightly-branch-name.yml

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

* Review feedback

* Add ability to skip all release steps

* Equal sign

* The right number of dashes

* Checkout before merge

* Checkout the default branch

* --no-ff

* git reset

* Skip publishing for artifacts which are not publishign to vcpkg

* Add git config --unset

* Apply suggestions from code review

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

* Review feedback

* Review feedback

* Remove extra reset

* TestRelease -> DailyRelease

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

eng/pipelines/daily-verification.yml

@@ -0,0 +1,54 @@
+
+jobs:
+  - job:
+    displayName: Daily vcpkg PR
+    pool:
+      name: azsdk-pool-mms-win-2019-general
+      vmImage: MMS2019
+
+    variables:
+      # The branch in the azure-sdk/vcpkg repo that is updated daily with the
+      # latest set of ports
+      TargetPrBranch: release/azure-sdk-for-cpp-daily-verification
+
+    steps:
+      - checkout: self
+
+      - template: /eng/pipelines/templates/steps/generate-daily-branch-name.yml
+
+      - template: /eng/pipelines/templates/steps/vcpkg-clone.yml
+        parameters:
+          Workspace: $(Pipeline.Workspace)
+          RepoOwner: azure-sdk
+
+      - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+        parameters:
+          WorkingDirectory: $(Pipeline.Workspace)/vcpkg
+
+      - pwsh: |
+          Write-Host "git checkout $(DailyBranchName)"
+          git checkout $(DailyBranchName)
+          if ($LASTEXITCODE -ne 0) {
+            Write-Error "Cannot check out $(DailyBranchName) LASTEXITCODE=$LASTEXITCODE"
+          }
+
+          Write-Host "##vso[task.setvariable variable=HasChanges]$true"
+        workingDirectory: $(Pipeline.Workspace)/vcpkg
+        displayName: Update vcpkg Daily Branch
+
+      # This step will skip creating the PR if the PR already exists
+      - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
+        parameters:
+          # Force push so this recreates the $(TargetPrBranch)
+          PushArgs: --force
+          BaseBranchName: $(DefaultBranch)
+          PRBranchName: $(TargetPrBranch)
+          RepoOwner: Microsoft
+          RepoName: vcpkg
+          WorkingDirectory: $(Pipeline.Workspace)/vcpkg
+          CommitMsg: Update with changes from $(DailyBranchName)
+          PRTitle: "[DO NOT MERGE] Daily Azure SDK for C++ CI validation"
+          PRBody: This is a long-lived draft PR should not be merged. Its branch is be updated periodically to validate the Azure SDK for C++ against the vcpkg CI.
+          ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts
+          OpenAsDraft: true
+          SkipCheckingForChanges: true

eng/pipelines/templates/stages/archetype-cpp-release.yml

@@ -1,6 +1,7 @@
 parameters:
   Artifacts: []
   ArtifactName: 'not-specified'
+  ServiceDirectory: not-specified
   TestPipeline: false
 
 stages:
@@ -9,7 +10,7 @@ stages:
       - stage: Release_${{ replace(artifact.Name, '-', '_') }}
         displayName: 'Release ${{artifact.name}}'
         dependsOn: Build
-        condition:  and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp'))
+        condition:  and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp'), ne(variables['Skip.AllRelease'], 'true'))
 
         jobs:
           - deployment: TagRepository
@@ -122,52 +123,18 @@ stages:
                           Write-Host "##vso[task.setvariable variable=PrTitle]$title"
                         displayName: Set PR title
 
-                      - task: Powershell@2
-                        inputs:
-                          filePath: eng/scripts/Initialize-VcpkgRelease.ps1
-                          arguments: >-
-                            -GitHubRepo "$(Build.Repository.Name)"
-                            -PackageSpecPath $(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json
-                            -SourceDirectory $(Pipeline.Workspace)/packages/${{artifact.Name}}/vcpkg
-                            -Verbose
-                          pwsh: true
-                        displayName: Initialize vcpkg release
-
                       # There are potential race conditions if this script runs
                       # in parallel against the same branch name. Release only 
                       # one package at a time.
-                      - pwsh: |
-                          Write-Host "git clone https://github.com/azure-sdk/vcpkg $(Pipeline.Workspace)/vcpkg"
-                          git clone https://github.com/azure-sdk/vcpkg $(Pipeline.Workspace)/vcpkg
-                          if ($LASTEXITCODE -ne 0) {
-                            Write-Error "Unable to check out vcpkg fork repo"
-                            exit $LASTEXITCODE
-                          }
-                          Write-Host "##vso[task.setvariable variable=VcpkgWorkspace]$(Pipeline.Workspace)/vcpkg"
-                        displayName: Clone vcpkg from upstream
+                      - template: /eng/pipelines/templates/steps/vcpkg-clone.yml
+                        parameters:
+                          RepoOwner: azure-sdk
+                          PRBranchName: $(PrBranchName)
 
-                      # Check out the PR branch if it's already in remote. 
-                      # Ignore failures.
-                      - pwsh: |
-                          $ErrorActionPreference = "Continue"
-                          git checkout "origin/$(PrBranchName)" 2>&1 | Out-Null
-                          $LASTEXITCODE = 0 # This ignores any error from git checkout
-                          git status
-                        displayName: Checkout Previous PRBranch if it exist.
-                        workingDirectory: $(VcpkgWorkspace)
-
-                      - task: Powershell@2
-                        inputs:
-                          pwsh: true
-                          targetType: filePath
-                          filePath: eng/scripts/Update-VcpkgPort.ps1
-                          arguments: >-
-                            -ReleaseArtifactSourceDirectory "$(Pipeline.Workspace)/packages/${{ artifact.Name }}"
-                            -PortDestinationDirectory 'ports/${{ artifact.VcpkgPortName }}'
-                            -VcpkgPortName '${{ artifact.VcpkgPortName }}'
-                            -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"'
-                          workingDirectory: $(VcpkgWorkspace)
-                        displayName: Update vcpkg port
+                      - template: /eng/pipelines/templates/steps/vcpkg-publish.yml
+                        parameters:
+                          ArtifactName: ${{ artifact.Name }}
+                          VcpkgPortName: ${{ artifact.VcpkgPortName }}
 
                       # Set $(HasChanges) to $true so that 
                       # create-pull-request.yml completes the push and PR 
@@ -214,4 +181,54 @@ stages:
                           PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId)
                           CommitMsg: "Increment package version after release of ${{ artifact.name }}"
                           PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases"
-                          CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}'
+                          CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}'
+
+  - ${{if eq(variables['System.TeamProject'], 'internal') }}:
+    - stage: Integration
+      dependsOn: Build
+      condition: and(succeeded(), or(eq(variables['PublishDailyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule')))
+      jobs:
+        - job: PublishDailyVcpkg
+          displayName: Publish to vcpkg daily branch
+
+          pool:
+            name: azsdk-pool-mms-win-2019-general
+            vmImage: MMS2019
+
+          steps:
+            - checkout: self
+            - download: current
+              artifact: packages
+
+            - template: /eng/pipelines/templates/steps/generate-daily-branch-name.yml
+
+            - template: /eng/pipelines/templates/steps/vcpkg-clone.yml
+              parameters:
+                RepoOwner: azure-sdk
+                PRBranchName: $(DailyBranchName)
+
+            # Placing artifacts in the same branch in sequence reduces the
+            # potential for race conditions. If this job does fail to push
+            # because another job has already pushed to the same branch it
+            # can be re-run.
+            - ${{ each artifact in parameters.Artifacts }}:
+              # Only run vcpkg publish if the artifact has a "VcpkgPortName"
+              # property. Absence of VcpkgPortName implies that the artifact 
+              # does not ship to vcpkg.
+              - ${{ if ne(artifact.VcpkgPortName, '') }}:
+                - template: /eng/pipelines/templates/steps/vcpkg-publish.yml
+                  parameters:
+                    ArtifactName: ${{ artifact.Name }}
+                    VcpkgPortName: ${{ artifact.VcpkgPortName }}
+                    DisplayNameExtension: ${{ artifact.Name }}
+                    DailyRelease: true
+                    DailyReleaseRef: $(Build.SourceVersion)
+
+            - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
+              parameters:
+                BaseRepoBranch: $(DailyBranchName)
+                CommitMsg: Daily vcpkg ports for ${{ parameters.ServiceDirectory }}
+                TargetRepoOwner: azure-sdk
+                TargetRepoName: vcpkg
+                WorkingDirectory: $(Pipeline.Workspace)/vcpkg
+                ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts

eng/pipelines/templates/steps/generate-daily-branch-name.yml

@@ -0,0 +1,9 @@
+steps:
+  - pwsh: |
+      $branchName = $env:DAILYBRANCHOVERRIDE
+      if (!$branchName) { 
+        $formattedDate = Get-Date -Format 'yyyy-MM-dd'
+        $branchName = "release/daily-$formattedDate"
+      }
+      Write-Host "##vso[task.setvariable variable=DailyBranchName]$branchName"
+    displayName: Set branch name in fork repo

eng/pipelines/templates/steps/vcpkg-clone.yml

@@ -0,0 +1,24 @@
+parameters: 
+  Workspace: $(Pipeline.Workspace)
+  RepoOwner: not-set
+  RepoName: vcpkg
+  PRBranchName: not-set
+
+steps: 
+  - pwsh: |
+      Write-Host "git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg"
+      git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg
+      if ($LASTEXITCODE -ne 0) {
+        Write-Error "Unable to check out vcpkg fork repo"
+        exit $LASTEXITCODE
+      }
+    displayName: Clone vcpkg from upstream
+
+  # Check out the PR branch if it's already in remote. Ignore failures.
+  - pwsh: |
+      $ErrorActionPreference = "Continue"
+      git checkout "origin/${{ parameters.PRBranchName }}" 2>&1 | Out-Null
+      $LASTEXITCODE = 0 # This ignores any error from git checkout
+      git status
+    displayName: Checkout Previous PRBranch if it exists
+    workingDirectory: ${{ parameters.Workspace }}/vcpkg

eng/pipelines/templates/steps/vcpkg-publish.yml

@@ -0,0 +1,35 @@
+parameters:
+  GitHubRepo: $(Build.Repository.Name)
+  Workspace: $(Pipeline.Workspace)
+  ArtifactName: not-set
+  VcpkgPortName: not-set
+  DisplayNameExtension: 
+  DailyReleaseRef: 
+  DailyRelease: false
+
+steps:
+  - task: Powershell@2
+    inputs:
+      filePath: eng/scripts/Initialize-VcpkgRelease.ps1
+      arguments: >-
+        -GitHubRepo "${{ parameters.GitHubRepo }}"
+        -PackageSpecPath ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/package-info.json
+        -SourceDirectory ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/vcpkg
+        -DailyReleaseRef '${{ parameters.DailyReleaseRef }}'
+        -Verbose
+      pwsh: true
+    displayName: Initialize vcpkg release ${{ parameters.DisplayNameExtension }}
+
+  - task: Powershell@2
+    inputs:
+      pwsh: true
+      targetType: filePath
+      filePath: eng/scripts/Update-VcpkgPort.ps1
+      arguments: >-
+        -ReleaseArtifactSourceDirectory "${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}"
+        -PortDestinationDirectory 'ports/${{ parameters.VcpkgPortName }}'
+        -VcpkgPortName '${{ parameters.VcpkgPortName }}'
+        -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"'
+        -DailyRelease:$${{ parameters.DailyRelease }}
+      workingDirectory: ${{ parameters.Workspace }}/vcpkg
+    displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }}

eng/scripts/Initialize-VcpkgRelease.ps1

@@ -12,8 +12,12 @@ Location of the relevant package-info.json file
 .PARAMETER GitHubRepo
 Name of the GitHub repo (of the form Azure/azure-sdk-for-cpp)
 
-#>
+.PARAMETER DailyReleaseRef
+If supplied update the portfile.cmake file's REF and SHA512 with values
+associated with the given ref.
 
+#>
+[CmdletBinding()]
 param (
     [Parameter(Mandatory = $true)]
     [ValidateNotNullOrEmpty()]
@@ -25,7 +29,9 @@ param (
 
     [Parameter(Mandatory = $true)]
     [ValidateNotNullOrEmpty()]
-    [string] $GitHubRepo
+    [string] $GitHubRepo,
+
+    [string] $DailyReleaseRef
 )
 
 # If there's nothing in the "port" folder to upload set SkipVcpkgUpdate to true
@@ -39,6 +45,11 @@ if (!(Get-ChildItem -Path "$SourceDirectory/port/CONTROL")) {
 $packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json
 $tarGzUri = "https://github.com/$GitHubRepo/archive/$($packageSpec.packageName).tar.gz" 
 
+if ($DailyReleaseRef) {
+    Write-Verbose "Initializing Daily Release"
+    $tarGzUri =  "https://github.com/$GitHubRepo/archive/$DailyReleaseRef.tar.gz"    
+}
+
 Write-Host "Downloading tarball to compute hash from $tarGzUri" 
 $localTarGzPath = New-TemporaryFile
 Invoke-WebRequest -Uri $tarGzUri -OutFile $localTarGzPath
@@ -53,7 +64,12 @@ $portfileLocation = "$SourceDirectory/port/portfile.cmake"
 # recommended in vcpkg documentation
 # Before: "   SHA512   1"
 # After:  "   SHA512   f6cf1c16c52" 
-$newContent = Get-Content -Raw -Path $portfileLocation `
-    | ForEach-Object { $_ -replace '(SHA512\s+)1', "`${1}$sha512" }
+$portFileContent = Get-Content -Raw -Path $portfileLocation 
+$newContent = $portFileContent -replace '(SHA512\s+)1', "`${1}$sha512"
+
+if ($DailyReleaseRef) {
+    Write-Verbose "Overriding REF with test release ref: $DailyReleaseRef"
+    $newContent = $newContent -replace '(?m)^(\s+)REF azure.*$', "`${1}REF $DailyReleaseRef"
+}
 
 $newContent | Set-Content $portfileLocation -NoNewLine

eng/scripts/Update-VcpkgPort.ps1

@@ -11,9 +11,9 @@ param (
     [ValidateNotNullOrEmpty()]
     [string] $VcpkgPortName,
 
-    [Parameter(Mandatory = $true)]
-    [ValidateNotNullOrEmpty()]
-    [string] $GitCommitParameters
+    [string] $GitCommitParameters,
+
+    [switch] $DailyRelease
 )
 
 ."$PSScriptRoot/../common/scripts/common.ps1"
@@ -77,8 +77,13 @@ if ($LASTEXITCODE -ne 0) {
     exit 1
 }
 
-Write-Host "./vcpkg.exe x-add-version $VcpkgPortName"
-./vcpkg.exe x-add-version $VcpkgPortName
+$addVersionAdditionalParameters = ''
+if ($DailyRelease) { 
+    $addVersionAdditionalParameters = '--overwrite-version'
+}
+
+Write-Host "./vcpkg.exe x-add-version $VcpkgPortName $addVersionAdditionalParameters"
+./vcpkg.exe x-add-version $VcpkgPortName $addVersionAdditionalParameters
 
 if ($LASTEXITCODE -ne 0) { 
     Write-Error "Failed to run vcpkg x-add-version $VcpkgPortName"
@@ -89,30 +94,37 @@ if ($LASTEXITCODE -ne 0) {
 Write-Host "git reset HEAD^"
 git reset HEAD^
 
-# Grab content needed for commit message and place in a temporary file
-$packageVersion = (Get-Content $ReleaseArtifactSourceDirectory/package-info.json -Raw | ConvertFrom-Json).version
-$commitMessageFile = New-TemporaryFile
-$chagelogEntry = Get-ChangeLogEntryAsString `
-    -ChangeLogLocation $ReleaseArtifactSourceDirectory/CHANGELOG.md `
-    -VersionString $PackageVersion
+# Only perform the final commit if this is not a test release
+if (!$DailyRelease) { 
+    # Grab content needed for commit message and place in a temporary file
+    $packageVersion = (Get-Content $ReleaseArtifactSourceDirectory/package-info.json -Raw | ConvertFrom-Json).version
+    $commitMessageFile = New-TemporaryFile
+    $chagelogEntry = Get-ChangeLogEntryAsString `
+        -ChangeLogLocation $ReleaseArtifactSourceDirectory/CHANGELOG.md `
+        -VersionString $PackageVersion
 
-"[$VcpkgPortName] Update to $PackageVersion`n$chagelogEntry" `
-    | Set-Content $commitMessageFile
+    "[$VcpkgPortName] Update to $PackageVersion`n$chagelogEntry" `
+        | Set-Content $commitMessageFile
 
-Write-Host "Commit Message:"
-Write-host (Get-Content $commitMessageFile -Raw)
+    Write-Host "Commit Message:"
+    Write-host (Get-Content $commitMessageFile -Raw)
 
+    Write-Host "git add -A"
+    git add -A
 
-Write-Host "git add -A"
-git add -A
+    # Final commit using commit message from the temporary file. Using the file
+    # enables the commit message to be formatted properly without having to write
+    # code to escape certain characters that might appear in the changelog file.
+    Write-Host "git $GitCommitParameters commit --file $commitMessageFile"
+    "git $GitCommitParameters commit --file $commitMessageFile" `
+        | Invoke-Expression -Verbose `
+        | Write-Host
+
+    # Set $(HasChanges) to $true so that create-pull-request.yml completes the 
+    # push and PR submission steps
+    Write-Host "##vso[task.setvariable variable=HasChanges]$true"
+}
 
-# Final commit using commit message from the temporary file. Using the file
-# enables the commit message to be formatted properly without having to write
-# code to escape certain characters that might appear in the changelog file.
-Write-Host "git $GitCommitParameters commit --file $commitMessageFile"
-"git $GitCommitParameters commit --file $commitMessageFile" `
-    | Invoke-Expression -Verbose `
-    | Write-Host
 
 <# 
 .SYNOPSIS
@@ -147,4 +159,9 @@ Additional parameters to supply to the `git commit` command. These are useful
 in the context of Azure DevOps where the git client does not have a configured
 user.name and user.email.
 
+.PARAMETER DailyRelease
+In the case of a test release set this to ensure that the x-add-version step
+includes `--overwrite-version` to ensure daily packages are properly updated
+in the vcpkg repo.
+
 #>