Vcpkg Beta Automation (for real) (#3497)

* Script that automates updating vcpkg betas

* Wire up to pipeline

* Spelling, commentary

* Clone into repo name

* Remove-Item requires the path to exist, even if -Force is specified

* Bootstrap vcpkg

* Arguments to Test-ShouldReleaseToVcpkg.ps1

* More commit

* Escape quotes

* Quote git commit

* Set strict mode

* Use tags

* Actually tag

* Run git commands

* Invoke-Expression

* Onboard releasing package to vcpkg before attempting to isntall from overaly

* temporarily disable matrix generation (30s tasks sometimes runs for 20+ minutes)

* Test-ShouldReleaseToVcpkg.ps1

* Test-Path

* Test vcpkg build, also ensure we push

* Just the tags... low probabiltiy of success given that '--' if frequently treated as a separator

* Add ability to skip link verification for tighter inner loop

* Push tag separately

* Unnecessary condition and conjunction

* Base repo owner: Azure

* Specify remote name

* Tighten inner loop

* git remote -v

* workingDirectory

* Go back to two tags and a regular push

* Saving some progress

* Use original push method

* BaseRepoOwner

* Cleanup, put changes in the default branch (vcpkg does not look at refs outside of the default branch)

* Use default branch

* Revert changes used to tighten up devops inner loop

* Revert an unrelated change

* Revert unrelated changes

* Revert unrelated changes

* Add spelling

.vscode/cspell.json

@@ -144,6 +144,8 @@
       "words": [
         "TEAMPROJECTID",
         "issecret",
+        "PSScriptRoot",
+        "BUILDID",
         "cppvcpkgcache"
       ]
     },

eng/pipelines/templates/stages/archetype-cpp-release.yml

@@ -123,7 +123,7 @@ stages:
                       - pwsh: |
                           $title = "[${{ artifact.VcpkgPortName }}] publish version $(PackageVersion)"
 
-                          if ('$(VcpkgPRTitle)') { 
+                          if ('$(VcpkgPRTitle)') {
                             Write-Host "Using queue time PR title"
                             $title = '$(VcpkgPRTitle)'
                           }
@@ -131,22 +131,46 @@ stages:
                         displayName: Set PR title
 
                       # There are potential race conditions if this script runs
-                      # in parallel against the same branch name. Release only 
+                      # in parallel against the same branch name. Release only
                       # one package at a time.
+                      # Clone main vcpkg repo
                       - template: /eng/pipelines/templates/steps/vcpkg-clone.yml
                         parameters:
                           RepoOwner: azure-sdk
                           PRBranchName: $(PrBranchName)
 
+                      # Clone Vcpkg Betas
+                      - template: /eng/pipelines/templates/steps/vcpkg-clone.yml
+                        parameters:
+                          RepoOwner: Azure
+                          RepoName: azure-sdk-vcpkg-betas
+
                       - template: /eng/pipelines/templates/steps/vcpkg-publish.yml
                         parameters:
                           ArtifactName: ${{ artifact.Name }}
                           VcpkgPortName: ${{ artifact.VcpkgPortName }}
 
-                      # Set $(HasChanges) to $true so that 
-                      # create-pull-request.yml completes the push and PR 
-                      # submission steps
+
+                      - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+                        parameters:
+                          WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-vcpkg-betas
+
+                      # Push changes to vcpkg betas
+                      - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
+                        parameters:
+                          WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-vcpkg-betas
+                          TargetRepoName: azure-sdk-vcpkg-betas
+                          BaseRepoOwner: Azure
+                          CommitMsg: Update vcpkg-configuration.json
+                          BaseRepoBranch: $(DefaultBranch)
+
+                      # Set $(HasChanges) to $true so that
+                      # create-pull-request.yml completes the push and PR
+                      # submission steps. This is contegnent upon
+                      # `$(PublishToVcpkg)` being `true`. `$(PublishToVcpkg)` is
+                      # set in `vcpkg-publish.yml`
                       - pwsh: Write-Host "##vso[task.setvariable variable=HasChanges]$true"
+                        condition: and(succeeded(), eq(variables['PublishToVcpkg'], 'true'))
                         displayName: Set $(HasChanges) to $true for create-pull-request.yml
 
                       - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
@@ -186,6 +210,7 @@ stages:
                             -RepoName $(VcpkgPrRepoName) `
                             -IssueNumber "$(Submitted.PullRequest.Number)" `
                             -Comment $prComment
+                        condition: and(succeeded(), eq(variables['PublishToVcpkg'], 'true'))
                         displayName: Comment notification to PR
 
           - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}:
@@ -244,7 +269,7 @@ stages:
             # can be re-run.
             - ${{ each artifact in parameters.Artifacts }}:
               # Only run vcpkg publish if the artifact has a "VcpkgPortName"
-              # property. Absence of VcpkgPortName implies that the artifact 
+              # property. Absence of VcpkgPortName implies that the artifact
               # does not ship to vcpkg.
               - ${{ if ne(artifact.VcpkgPortName, '') }}:
                 - template: /eng/pipelines/templates/steps/vcpkg-publish.yml
@@ -254,6 +279,7 @@ stages:
                     DisplayNameExtension: ${{ artifact.Name }}
                     DailyRelease: true
                     DailyReleaseRef: $(Build.SourceVersion)
+                    UpdateVcpkgBeta: false
 
             - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
               parameters:

eng/pipelines/templates/steps/vcpkg-clone.yml

@@ -4,15 +4,15 @@ parameters:
   RepoName: vcpkg
   PRBranchName: not-set
 
-steps: 
+steps:
   - pwsh: |
-      Write-Host "git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg"
-      git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg
+      Write-Host "git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/${{ parameters.RepoName }}"
+      git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/${{ parameters.RepoName }}
       if ($LASTEXITCODE -ne 0) {
         Write-Error "Unable to check out vcpkg fork repo"
         exit $LASTEXITCODE
       }
-    displayName: Clone vcpkg from upstream
+    displayName: Clone vcpkg (${{ parameters.RepoOwner }}/${{ parameters.RepoName }})
 
   # Check out the PR branch if it's already in remote. Ignore failures.
   - pwsh: |

eng/pipelines/templates/steps/vcpkg-publish.yml

@@ -3,9 +3,10 @@ parameters:
   Workspace: $(Pipeline.Workspace)
   ArtifactName: not-set
   VcpkgPortName: not-set
-  DisplayNameExtension: 
-  DailyReleaseRef: 
+  DisplayNameExtension:
+  DailyReleaseRef:
   DailyRelease: false
+  UpdateVcpkgBeta: true
 
 steps:
   - task: Powershell@2
@@ -20,6 +21,19 @@ steps:
       pwsh: true
     displayName: Initialize vcpkg release ${{ parameters.DisplayNameExtension }}
 
+  # In some cases there should be no release to vcpkg. If vcpkg should be
+  # released, set "PublishToVcpkg" to "true" so we can make changes.
+  - task: Powershell@2
+    inputs:
+      pwsh: true
+      targetType: filePath
+      filePath: eng/scripts/Test-ShouldReleaseToVcpkg.ps1
+      arguments: >-
+        -ReleaseArtifactSourceDirectory "${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}"
+        -VcpkgFolder ${{ parameters.Workspace }}/vcpkg
+        -VcpkgPortName '${{ parameters.VcpkgPortName }}'
+    displayName: Check whether to release to vcpkg
+
   - task: Powershell@2
     inputs:
       pwsh: true
@@ -32,4 +46,20 @@ steps:
         -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"'
         -DailyRelease:$${{ parameters.DailyRelease }}
       workingDirectory: ${{ parameters.Workspace }}/vcpkg
-    displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }}
+      condition: and(succeeded(), eq(variables['PublishToVcpkg'], 'true'))
+    displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }}
+
+  # On package release vcpkg beta should always be updated
+  - ${{ if eq(parameters.UpdateVcpkgBeta, 'true') }}:
+    - task: Powershell@2
+      inputs:
+        pwsh: true
+        targetType: filePath
+        filePath: eng/scripts/Update-VcpkgBeta.ps1
+        arguments: >-
+          -VcpkgBetaFolder ${{ parameters.Workspace }}/azure-sdk-vcpkg-betas
+          -VcpkgFolder ${{ parameters.Workspace }}/vcpkg
+          -ReleaseArtifactSourceDirectory "${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}"
+          -VcpkgPortName '${{ parameters.VcpkgPortName }}'
+          -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"'
+      displayName: Update Vcpkg Betas port ${{ parameters.DisplayNameExtension }}

eng/scripts/Test-ShouldReleaseToVcpkg.ps1

@@ -0,0 +1,47 @@
+param(
+    [string] $ReleaseArtifactSourceDirectory,
+    [string] $VcpkgFolder,
+    [string] $VcpkgPortName
+)
+
+."$PSSCriptRoot/../common/scripts/common.ps1"
+
+Set-StrictMode -Version 3
+
+$packageJsonContents = Get-Content `
+    -Path "$ReleaseArtifactSourceDirectory/package-info.json" `
+    -Raw
+$packageJson = ConvertFrom-Json $packageJsonContents
+$packageVersionSemver = [AzureEngSemanticVersion]::ParseVersionString($packageJson.version)
+
+if (!$packageVersionSemver.IsPrerelease) {
+    Write-Host "Package version is GA ($($packageJson.version)), publish to vcpkg"
+    Write-Host "##vso[task.setvariable variable=PublishToVcpkg]true"
+    exit 0
+}
+Write-Host "Released package is preview"
+
+# The package does not exist in vcpkg, publish to vcpkg
+$vcpkgJsonPath = "$VcpkgFolder/ports/$VcpkgPortName/vcpkg.json"
+if (!(Test-Path $vcpkgJsonPath)) {
+    Write-Host "Package ($VcpkgPortName) has not been published, publish to vcpkg"
+    Write-Host "##vso[task.setvariable variable=PublishToVcpkg]true"
+    exit 0
+}
+Write-Host "Package has been published before"
+
+$vcpkgJsonContents = Get-Content -Raw -Path $vcpkgJsonPath
+$vcpkgJson = ConvertFrom-Json $vcpkgJsonContents
+
+$existingVersion = [AzureEngSemanticVersion]::ParseVersionString($vcpkgJson.'version-semver')
+
+# Published version is a prerelease
+if ($existingVersion.IsPrerelease) {
+    Write-Host "Existing version ($($vcpkgJson.'version-semver')) is prerelease, publish to vcpkg"
+    Write-Host "##vso[task.setvariable variable=PublishToVcpkg]true"
+    exit 0
+}
+Write-Host "Existing version is GA"
+
+Write-Host "Criteria for publishing not satisifed, do NOT publish to vcpkg"
+Write-Host "##vso[task.setvariable variable=PublishToVcpkg]false"

eng/scripts/Update-VcpkgBeta.ps1

@@ -0,0 +1,89 @@
+param(
+    [string] $VcpkgBetaFolder,
+    [string] $VcpkgFolder,
+    [string] $ReleaseArtifactSourceDirectory,
+    [string] $VcpkgPortName,
+    [string] $GitCommitParameters,
+    [string] $BuildIdentifier = $env:BUILD_BUILDID
+)
+
+."$PSSCriptRoot/../common/scripts/common.ps1"
+Set-StrictMode -Version 3
+
+# To ensure a clean synchronization remove all files at the destination.
+# This ensures that files no longer present in the build output do not
+# persist in later versions.
+$portFolder = "$VcpkgBetaFolder/ports/$VcpkgPortName"
+if (Test-Path $portFolder) {
+    Remove-Item $portFolder -Recurse -Force
+}
+New-Item -ItemType Directory -Path $portFolder
+
+Copy-Item `
+    -Path "$ReleaseArtifactSourceDirectory/vcpkg/port/*" `
+    -Destination $portFolder
+
+$rawPackageInfo = Get-Content -Raw -Path $ReleaseArtifactSourceDirectory/package-info.json
+$packageInfo = ConvertFrom-Json $rawPackageInfo
+
+$originalLocation = Get-Location
+try {
+    Set-Location $VcpkgFolder
+    if ($IsWindows) {
+        Write-Host "./bootstrap-vcpkg.bat"
+        ./bootstrap-vcpkg.bat
+    } else {
+        Write-Host "./bootstrap-vcpkg.sh"
+        ./bootstrap-vcpkg.sh
+    }
+
+    Set-Location $VcpkgBetaFolder
+
+    Write-Host "$VcpkgFolder/vcpkg format-manifest --all --vcpkg-root=. --x-scripts-root=$VcpkgFolder/scripts"
+    & $VcpkgFolder/vcpkg format-manifest `
+        --all `
+        --vcpkg-root=. `
+        --x-scripts-root=$VcpkgFolder/scripts
+
+    Write-Host "git add -A"
+    git add -A
+    Write-Host "git $GitCommitParameters commit -m `"$(Get-Date -Format "yyyy-MM-dd" ): $VcpkgPortName $($packageInfo.version)`""
+    Invoke-Expression "git $GitCommitParameters commit -m `"$(Get-Date -Format "yyyy-MM-dd" ): $VcpkgPortName $($packageInfo.version)`""
+
+    Write-Host "$VcpkgFolder/vcpkg x-add-version $VcpkgPortName --vcpkg-root=. --x-scripts-root=$VcpkgFolder/scripts"
+    & $VcpkgFolder/vcpkg x-add-version `
+        $VcpkgPortName `
+        --vcpkg-root=. `
+        --x-scripts-root=$VcpkgFolder/scripts
+
+    Write-Host "git add -A"
+    git add -A
+    Write-Host "git $GitCommitParameters commit --amend --no-edit"
+    Invoke-Expression "git $GitCommitParameters commit --amend --no-edit"
+
+    # WARNING: You may not use the normal
+    Write-Host "git log -1 --format=format:%H"
+    $baseHash = git log -1 --format=format:%H
+    Write-Host "New Baseline: $baseHash"
+
+    # Update vcpkg-configuration.json to include this package and set the
+    # baseline
+    $vcpkgConfigPath = "$VcpkgBetaFolder/vcpkg-configuration.json"
+    $rawVcpkgConfig = Get-Content -Raw -Path $vcpkgConfigPath
+    $vcpkgConfig = ConvertFrom-Json $rawVcpkgConfig
+
+    $vcpkgConfig.registries[0].baseline = $baseHash
+    if (!($vcpkgConfig.registries[0].packages -contains $VcpkgPortName)) {
+        $vcpkgConfig.registries[0].packages += $VcpkgPortName
+    }
+
+    $vcpkgConfigJson = ConvertTo-Json $vcpkgConfig -Depth 100
+    Set-Content -Path $vcpkgConfigPath -Value $vcpkgConfigJson
+
+    Write-Host "git add -A"
+    git add -A
+    Write-Host "git $GitCommitParameters commit -m `"Update vcpkg-configuration.json`""
+    "git $GitCommitParameters commit -m 'Update vcpkg-configuration.json'"
+} finally {
+    Set-Location $originalLocation
+}