diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index d24656dc3..d12347900 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -34,25 +34,25 @@ parameters: default: [] jobs: - - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml - parameters: - MatrixConfigs: - - Name: base - Path: eng/pipelines/templates/stages/platform-matrix.json - Selection: all - GenerateVMJobs: true - JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml - AdditionalParameters: - Artifacts: ${{ parameters.Artifacts }} - ServiceDirectory: ${{ parameters.ServiceDirectory }} - TestPipeline: ${{ parameters.TestPipeline }} - CtestRegex: ${{ parameters.CtestRegex }} - CtestExcludeRegex: ${{ parameters.CtestExcludeRegex }} - CoverageReportPath: ${{ parameters.CoverageReportPath }} - CoverageEnabled: ${{ parameters.CoverageEnabled }} - LineCoverageTarget: ${{ parameters.LineCoverageTarget }} - BranchCoverageTarget: ${{ parameters.BranchCoverageTarget }} - TestEnv: ${{ parameters.TestEnv }} + # - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml + # parameters: + # MatrixConfigs: + # - Name: base + # Path: eng/pipelines/templates/stages/platform-matrix.json + # Selection: all + # GenerateVMJobs: true + # JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml + # AdditionalParameters: + # Artifacts: ${{ parameters.Artifacts }} + # ServiceDirectory: ${{ parameters.ServiceDirectory }} + # TestPipeline: ${{ parameters.TestPipeline }} + # CtestRegex: ${{ parameters.CtestRegex }} + # CtestExcludeRegex: ${{ parameters.CtestExcludeRegex }} + # CoverageReportPath: ${{ parameters.CoverageReportPath }} + # CoverageEnabled: ${{ parameters.CoverageEnabled }} + # LineCoverageTarget: ${{ parameters.LineCoverageTarget }} + # BranchCoverageTarget: ${{ parameters.BranchCoverageTarget }} + # TestEnv: ${{ parameters.TestEnv }} # Disable build for cpp - client - ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}: @@ -64,14 +64,15 @@ jobs: VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' Package.EnableSBOMSigning: true steps: - - template: /eng/common/pipelines/templates/steps/check-spelling.yml - parameters: - ContinueOnError: false + # - template: /eng/common/pipelines/templates/steps/check-spelling.yml + # parameters: + # ContinueOnError: false - - template: /eng/common/pipelines/templates/steps/verify-links.yml - parameters: - Directory: '' - CheckLinkGuidance: $true + # - template: /eng/common/pipelines/templates/steps/verify-links.yml + # parameters: + # Directory: '' + # CheckLinkGuidance: $true + # Condition: and(succeeded(), ne(variables['Skip.LinkVerification'], 'true')) - ${{ each artifact in parameters.Artifacts }}: - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml @@ -108,8 +109,8 @@ jobs: -DBUILD_TRANSPORT_CURL=OFF -DBUILD_DOCUMENTATION=YES - - pwsh: npm install -g moxygen - displayName: Install Moxygen to generate markdown for docs.microsoft.com + # - pwsh: npm install -g moxygen + # displayName: Install Moxygen to generate markdown for docs.microsoft.com # Generate package spec JSON files for tagging the repo - ${{ each artifact in parameters.Artifacts }}: @@ -167,17 +168,17 @@ jobs: workingDirectory: build displayName: Generate docs (${{ artifact.Name }}-docs) - - task: PowerShell@2 - inputs: - targetType: filePath - filePath: eng/scripts/New-DocsMsArtifact.ps1 - arguments: >- - -ServiceDirectory ${{ parameters.ServiceDirectory }} - -PackageName ${{ artifact.Name }} - -TargetFolder $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/docs/docs.ms - ignoreLASTEXITCODE: true - pwsh: true - displayName: Generate artifacts for docs.ms + # - task: PowerShell@2 + # inputs: + # targetType: filePath + # filePath: eng/scripts/New-DocsMsArtifact.ps1 + # arguments: >- + # -ServiceDirectory ${{ parameters.ServiceDirectory }} + # -PackageName ${{ artifact.Name }} + # -TargetFolder $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/docs/docs.ms + # ignoreLASTEXITCODE: true + # pwsh: true + # displayName: Generate artifacts for docs.ms - pwsh: | New-Item -ItemType directory -Path $(Build.ArtifactStagingDirectory) -Name docs/${{ artifact.Name }} @@ -219,15 +220,15 @@ jobs: artifactName: docs path: $(Build.ArtifactStagingDirectory)/docs - - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml + # - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate BOM' - condition: succeededOrFailed() - inputs: - BuildDropPath: $(Build.SourcesDirectory)/build + # - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + # displayName: 'Generate BOM' + # condition: succeededOrFailed() + # inputs: + # BuildDropPath: $(Build.SourcesDirectory)/build - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactPath: '$(Build.SourcesDirectory)/build/_manifest' - ArtifactName: 'release_artifact_manifest' + # - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + # parameters: + # ArtifactPath: '$(Build.SourcesDirectory)/build/_manifest' + # ArtifactName: 'release_artifact_manifest' diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index e7709d27c..21f35c8ad 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -123,7 +123,7 @@ stages: - pwsh: | $title = "[${{ artifact.VcpkgPortName }}] publish version $(PackageVersion)" - if ('$(VcpkgPRTitle)') { + if ('$(VcpkgPRTitle)') { Write-Host "Using queue time PR title" $title = '$(VcpkgPRTitle)' } @@ -131,22 +131,46 @@ stages: displayName: Set PR title # There are potential race conditions if this script runs - # in parallel against the same branch name. Release only + # in parallel against the same branch name. Release only # one package at a time. + # Clone main vcpkg repo - template: /eng/pipelines/templates/steps/vcpkg-clone.yml parameters: RepoOwner: azure-sdk PRBranchName: $(PrBranchName) + # Clone Vcpkg Betas + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: Azure + RepoName: azure-sdk-vcpkg-betas + - template: /eng/pipelines/templates/steps/vcpkg-publish.yml parameters: ArtifactName: ${{ artifact.Name }} VcpkgPortName: ${{ artifact.VcpkgPortName }} - # Set $(HasChanges) to $true so that - # create-pull-request.yml completes the push and PR - # submission steps + + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + parameters: + WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-vcpkg-betas + + # Push changes to vcpkg betas + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-vcpkg-betas + TargetRepoName: azure-sdk-vcpkg-betas + BaseRepoOwner: Azure + CommitMsg: Update vcpkg-configuration.json + BaseRepoBranch: $(DefaultBranch) + + # Set $(HasChanges) to $true so that + # create-pull-request.yml completes the push and PR + # submission steps. This is contegnent upon + # `$(PublishToVcpkg)` being `true`. `$(PublishToVcpkg)` is + # set in `vcpkg-publish.yml` - pwsh: Write-Host "##vso[task.setvariable variable=HasChanges]$true" + condition: and(succeeded(), eq(variables['PublishToVcpkg'], 'true')) displayName: Set $(HasChanges) to $true for create-pull-request.yml - template: /eng/common/pipelines/templates/steps/set-default-branch.yml @@ -186,6 +210,7 @@ stages: -RepoName $(VcpkgPrRepoName) ` -IssueNumber "$(Submitted.PullRequest.Number)" ` -Comment $prComment + condition: and(succeeded(), eq(variables['PublishToVcpkg'], 'true')) displayName: Comment notification to PR - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}: @@ -244,7 +269,7 @@ stages: # can be re-run. - ${{ each artifact in parameters.Artifacts }}: # Only run vcpkg publish if the artifact has a "VcpkgPortName" - # property. Absence of VcpkgPortName implies that the artifact + # property. Absence of VcpkgPortName implies that the artifact # does not ship to vcpkg. - ${{ if ne(artifact.VcpkgPortName, '') }}: - template: /eng/pipelines/templates/steps/vcpkg-publish.yml @@ -254,6 +279,7 @@ stages: DisplayNameExtension: ${{ artifact.Name }} DailyRelease: true DailyReleaseRef: $(Build.SourceVersion) + UpdateVcpkgBeta: false - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: diff --git a/eng/pipelines/templates/steps/vcpkg-clone.yml b/eng/pipelines/templates/steps/vcpkg-clone.yml index 94640bb22..b374bee15 100644 --- a/eng/pipelines/templates/steps/vcpkg-clone.yml +++ b/eng/pipelines/templates/steps/vcpkg-clone.yml @@ -4,15 +4,15 @@ parameters: RepoName: vcpkg PRBranchName: not-set -steps: +steps: - pwsh: | - Write-Host "git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg" - git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg + Write-Host "git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/${{ parameters.RepoName }}" + git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/${{ parameters.RepoName }} if ($LASTEXITCODE -ne 0) { Write-Error "Unable to check out vcpkg fork repo" exit $LASTEXITCODE } - displayName: Clone vcpkg from upstream + displayName: Clone vcpkg (${{ parameters.RepoOwner }}/${{ parameters.RepoName }}) # Check out the PR branch if it's already in remote. Ignore failures. - pwsh: | diff --git a/eng/pipelines/templates/steps/vcpkg-publish.yml b/eng/pipelines/templates/steps/vcpkg-publish.yml index 0493eebc8..dbf9f7cca 100644 --- a/eng/pipelines/templates/steps/vcpkg-publish.yml +++ b/eng/pipelines/templates/steps/vcpkg-publish.yml @@ -3,9 +3,10 @@ parameters: Workspace: $(Pipeline.Workspace) ArtifactName: not-set VcpkgPortName: not-set - DisplayNameExtension: - DailyReleaseRef: + DisplayNameExtension: + DailyReleaseRef: DailyRelease: false + UpdateVcpkgBeta: true steps: - task: Powershell@2 @@ -20,6 +21,19 @@ steps: pwsh: true displayName: Initialize vcpkg release ${{ parameters.DisplayNameExtension }} + # In some cases there should be no release to vcpkg. If vcpkg should be + # released, set "PublishToVcpkg" to "true" so we can make changes. + - task: Powershell@2 + inputs: + pwsh: true + targetType: filePath + filePath: eng/scripts/Test-ShouldReleaseToVcpkg.ps1 + arguments: >- + -ReleaseArtifactSourceDirectory "${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}" + -VcpkgFolder ${{ parameters.Workspace }}/vcpkg + -VcpkgPortName '${{ parameters.VcpkgPortName }}' + displayName: Check whether to release to vcpkg + - task: Powershell@2 inputs: pwsh: true @@ -32,4 +46,20 @@ steps: -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"' -DailyRelease:$${{ parameters.DailyRelease }} workingDirectory: ${{ parameters.Workspace }}/vcpkg - displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }} \ No newline at end of file + condition: and(succeeded(), eq(variables['PublishToVcpkg'], 'true')) + displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }} + + # On package release vcpkg beta should always be updated + - ${{ if eq(parameters.UpdateVcpkgBeta, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + targetType: filePath + filePath: eng/scripts/Update-VcpkgBeta.ps1 + arguments: >- + -VcpkgBetaFolder ${{ parameters.Workspace }}/azure-sdk-vcpkg-betas + -VcpkgFolder ${{ parameters.Workspace }}/vcpkg + -ReleaseArtifactSourceDirectory "${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}" + -VcpkgPortName '${{ parameters.VcpkgPortName }}' + -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"' + displayName: Update Vcpkg Betas port ${{ parameters.DisplayNameExtension }} diff --git a/eng/scripts/Test-ShouldReleaseToVcpkg.ps1 b/eng/scripts/Test-ShouldReleaseToVcpkg.ps1 new file mode 100644 index 000000000..57c6979d0 --- /dev/null +++ b/eng/scripts/Test-ShouldReleaseToVcpkg.ps1 @@ -0,0 +1,47 @@ +param( + [string] $ReleaseArtifactSourceDirectory, + [string] $VcpkgFolder, + [string] $VcpkgPortName +) + +."$PSSCriptRoot/../common/scripts/common.ps1" + +Set-StrictMode -Version 3 + +$packageJsonContents = Get-Content ` + -Path "$ReleaseArtifactSourceDirectory/package-info.json" ` + -Raw +$packageJson = ConvertFrom-Json $packageJsonContents +$packageVersionSemver = [AzureEngSemanticVersion]::ParseVersionString($packageJson.version) + +if (!$packageVersionSemver.IsPrerelease) { + Write-Host "Package version is GA ($($packageJson.version)), publish to vcpkg" + Write-Host "##vso[task.setvariable variable=PublishToVcpkg]true" + exit 0 +} +Write-Host "Released package is preview" + +# The package does not exist in vcpkg, publish to vcpkg +$vcpkgJsonPath = "$VcpkgFolder/ports/$VcpkgPortName/vcpkg.json" +if (!(Test-Path $vcpkgJsonPath)) { + Write-Host "Package ($VcpkgPortName) has not been published, publish to vcpkg" + Write-Host "##vso[task.setvariable variable=PublishToVcpkg]true" + exit 0 +} +Write-Host "Package has been published before" + +$vcpkgJsonContents = Get-Content -Raw -Path $vcpkgJsonPath +$vcpkgJson = ConvertFrom-Json $vcpkgJsonContents + +$existingVersion = [AzureEngSemanticVersion]::ParseVersionString($vcpkgJson.'version-semver') + +# Published version is a prerelease +if ($existingVersion.IsPrerelease) { + Write-Host "Existing version ($($vcpkgJson.'version-semver')) is prerelease, publish to vcpkg" + Write-Host "##vso[task.setvariable variable=PublishToVcpkg]true" + exit 0 +} +Write-Host "Existing version is GA" + +Write-Host "Criteria for publishing not satisifed, do NOT publish to vcpkg" +Write-Host "##vso[task.setvariable variable=PublishToVcpkg]false" \ No newline at end of file diff --git a/eng/scripts/Update-VcpkgBeta.ps1 b/eng/scripts/Update-VcpkgBeta.ps1 new file mode 100644 index 000000000..d626ea3b3 --- /dev/null +++ b/eng/scripts/Update-VcpkgBeta.ps1 @@ -0,0 +1,89 @@ +param( + [string] $VcpkgBetaFolder, + [string] $VcpkgFolder, + [string] $ReleaseArtifactSourceDirectory, + [string] $VcpkgPortName, + [string] $GitCommitParameters, + [string] $BuildIdentifier = $env:BUILD_BUILDID +) + +."$PSSCriptRoot/../common/scripts/common.ps1" +Set-StrictMode -Version 3 + +# To ensure a clean synchronization remove all files at the destination. +# This ensures that files no longer present in the build output do not +# persist in later versions. +$portFolder = "$VcpkgBetaFolder/ports/$VcpkgPortName" +if (Test-Path $portFolder) { + Remove-Item $portFolder -Recurse -Force +} +New-Item -ItemType Directory -Path $portFolder + +Copy-Item ` + -Path "$ReleaseArtifactSourceDirectory/vcpkg/port/*" ` + -Destination $portFolder + +$rawPackageInfo = Get-Content -Raw -Path $ReleaseArtifactSourceDirectory/package-info.json +$packageInfo = ConvertFrom-Json $rawPackageInfo + +$originalLocation = Get-Location +try { + Set-Location $VcpkgFolder + if ($IsWindows) { + Write-Host "./bootstrap-vcpkg.bat" + ./bootstrap-vcpkg.bat + } else { + Write-Host "./bootstrap-vcpkg.sh" + ./bootstrap-vcpkg.sh + } + + Set-Location $VcpkgBetaFolder + + Write-Host "$VcpkgFolder/vcpkg format-manifest --all --vcpkg-root=. --x-scripts-root=$VcpkgFolder/scripts" + & $VcpkgFolder/vcpkg format-manifest ` + --all ` + --vcpkg-root=. ` + --x-scripts-root=$VcpkgFolder/scripts + + Write-Host "git add -A" + git add -A + Write-Host "git $GitCommitParameters commit -m `"$(Get-Date -Format "yyyy-MM-dd" ): $VcpkgPortName $($packageInfo.version)`"" + Invoke-Expression "git $GitCommitParameters commit -m `"$(Get-Date -Format "yyyy-MM-dd" ): $VcpkgPortName $($packageInfo.version)`"" + + Write-Host "$VcpkgFolder/vcpkg x-add-version $VcpkgPortName --vcpkg-root=. --x-scripts-root=$VcpkgFolder/scripts" + & $VcpkgFolder/vcpkg x-add-version ` + $VcpkgPortName ` + --vcpkg-root=. ` + --x-scripts-root=$VcpkgFolder/scripts + + Write-Host "git add -A" + git add -A + Write-Host "git $GitCommitParameters commit --amend --no-edit" + Invoke-Expression "git $GitCommitParameters commit --amend --no-edit" + + # WARNING: You may not use the normal + Write-Host "git log -1 --format=format:%H" + $baseHash = git log -1 --format=format:%H + Write-Host "New Baseline: $baseHash" + + # Update vcpkg-configuration.json to include this package and set the + # baseline + $vcpkgConfigPath = "$VcpkgBetaFolder/vcpkg-configuration.json" + $rawVcpkgConfig = Get-Content -Raw -Path $vcpkgConfigPath + $vcpkgConfig = ConvertFrom-Json $rawVcpkgConfig + + $vcpkgConfig.registries[0].baseline = $baseHash + if (!($vcpkgConfig.registries[0].packages -contains $VcpkgPortName)) { + $vcpkgConfig.registries[0].packages += $VcpkgPortName + } + + $vcpkgConfigJson = ConvertTo-Json $vcpkgConfig -Depth 100 + Set-Content -Path $vcpkgConfigPath -Value $vcpkgConfigJson + + Write-Host "git add -A" + git add -A + Write-Host "git $GitCommitParameters commit -m `"Update vcpkg-configuration.json`"" + "git $GitCommitParameters commit -m 'Update vcpkg-configuration.json'" +} finally { + Set-Location $originalLocation +} diff --git a/eng/scripts/Update-VcpkgPort.ps1 b/eng/scripts/Update-VcpkgPort.ps1 index cb27ef3ea..806c511c6 100644 --- a/eng/scripts/Update-VcpkgPort.ps1 +++ b/eng/scripts/Update-VcpkgPort.ps1 @@ -29,7 +29,7 @@ if ((Get-Command git | Measure-Object).Count -eq 0) { exit 1 } -if (!(Test-Path $ReleaseArtifactSourceDirectory/package-info.json)) { +if (!(Test-Path $ReleaseArtifactSourceDirectory/package-info.json)) { LogError "Could not locate package-info.json in -ReleaseArtifactSourceDirectory" exit 1 } @@ -42,7 +42,7 @@ if (!(Test-Path $ReleaseArtifactSourceDirectory/vcpkg/port)) { LogError "Could not locate vcpkg/port directory in -ReleaseArtifactSourceDirectory" } -# Clean out the folder so that template files removed are not inadvertently +# Clean out the folder so that template files removed are not inadvertently # re-added if (Test-Path $PortDestinationDirectory) { Remove-Item -v -r $PortDestinationDirectory @@ -124,7 +124,7 @@ if (!$DailyRelease) { | Invoke-Expression -Verbose ` | Write-Host - # Set $(HasChanges) to $true so that create-pull-request.yml completes the + # Set $(HasChanges) to $true so that create-pull-request.yml completes the # push and PR submission steps Write-Host "##vso[task.setvariable variable=HasChanges]$true" } @@ -137,7 +137,7 @@ Uses release artifacts to update a vcpkg port .DESCRIPTION This script updates a given vcpkg port using C++ release artifacts. It requires that the GitHub repo is tagged and a release is available at that tag for -generating the SHA of the vcpkg artifact. +generating the SHA of the vcpkg artifact. This script also uses the contents of the changelog at the release version in the commit message. diff --git a/sdk/core/CMakeLists.txt b/sdk/core/CMakeLists.txt index 66c41f888..17e0225f6 100644 --- a/sdk/core/CMakeLists.txt +++ b/sdk/core/CMakeLists.txt @@ -9,7 +9,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) add_subdirectory(azure-core) -if (BUILD_PERFORMANCE_TESTS) +if (BUILD_PERFORMANCE_TESTS) add_subdirectory(perf) endif()