From c534a1e16004346b574242b58e54728a522b9456 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 22 Oct 2021 13:49:10 -0700 Subject: [PATCH] Cognitive Matrix Consolidation (#2947) * Refactor template to use multiple builds on the same machine resetting the machine and runtime varaibles along the way Move template expression out of default matrix parameters Remove extra reference to ci/jobs.yml Remove unnecessary ci.yml, change param declaration in devops-variables-set.yml Heredoc Fix set/reset scripts after testing with devops outputs -ErrorAction Ignore Smaller matrix OsVMImage comes from machine, not parameters Add Ubuntu 20 to matrix Remove checkout: self, it is implied Move machine-setup steps outside of loop CODE_COVERAGE handling tab Remove creation of CODE_COVERAGE Use AptDependencies and add apt update to dependency install step Remove extra preparation steps Enable code coverage check scenario Add MacOS 11 Disable vcpkg cache Coverage configuration Pass parameter properly bool -> boolean Install coverage tools if it's possible that coverage may run More logging on code coverage Show contents of Makefile One vcpkg to build them all Verbose SourcesDirectory Nesting VcpkgInstall workingDirectory for vcpkg install AptDependencies complete the conditional Enable all matrix entries sudo Remove extra chatter More scrubbing post-build to help with Windows scenarios sudo echo Get-ChildItem ErrorAction sudo sh -c More ErrorAction output sourceListFile Disable additional sources.list Install coverage tools when used for coverage COndition Do not build testing for Windows arm64 Correct path for coverage tools ne -> eq Add more scenarios to the matrix Remove libxml2 Add MacOS 10.15 to matrix Revert "Remove libxml2" This reverts commit 178af8b89abb21ee84dbcd25b4d54bd90ec1d44c. Add libxml2 to MacOS 10.15 matrix entry Display name decorator, refine the matrix Remove OsVmImage variable Add commentary to tests about matrix Rebalance matrix, nest EnvVars, DisplayNameDecorator Correct parameter access AptDependencies Ensure building for WindowsStore in UWP builds, Only build tests for debug builds in altered matrix entires, Linux build type specified as environment variable Rebalance matrix for better timing Remove all instances of libxml2 in vcpkg dependencies Move common devops tasks to eng/common Set eng/common paths in pipelines Revert "Remove all instances of libxml2 in vcpkg dependencies" This reverts commit 35ffbc7aac87f400fdf3351076346750c1c750ed. https://github.com/Azure/azure-sdk-for-cpp/issues/2946 Permit words like iname in more pipeline .yml files Remove libxml2 from vcpkg dependencies for Windows Perf test matrix maxParallel: 10 Validate apt install first apt install first maxParallel: 12 -j 8 -j 10 Use matrix generator Add expected parameters Move parameters to appropriate file, fix cloud spec Specify correct platform matrix location DependsOn Platform matrix Remove DisplayNameDecorator, try more clang information Rapid iteration matrix clang --version install clang-11 Use correct pool Use full matrix, revert additional logic in jobs/ci.yml * Review feedback: newline * Review feedback: Test in release, too * Remove DisplayNameDecorator (it was useful when we were putting multiple builds into a single job, now it is redundant) * Remove displayNames * Matrix review feedback * Conditions * Rename ci.yml -> ci.tests.yml --- .vscode/cspell.json | 2 +- .../templates/jobs/archetype-sdk-client.yml | 544 +++++------------- .../templates/jobs/archetype-sdk-tests.yml | 11 +- eng/pipelines/templates/jobs/ci.tests.yml | 138 +++++ .../templates/stages/archetype-sdk-client.yml | 8 +- .../templates/stages/platform-matrix.json | 154 +++++ eng/pipelines/templates/steps/vcpkg.yml | 3 +- sdk/storage/ci.yml | 2 +- 8 files changed, 454 insertions(+), 408 deletions(-) create mode 100644 eng/pipelines/templates/jobs/ci.tests.yml create mode 100644 eng/pipelines/templates/stages/platform-matrix.json diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 4501c1f8c..476b29d62 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -113,7 +113,7 @@ ], "overrides": [ { - "filename": "**/eng/pipelines/templates/jobs/archetype-sdk-client.yml", + "filename": "**/eng/pipelines/templates/**/*.yml", "words": [ "lcov", "iname" diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 2794568b4..5dfa64140 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -8,9 +8,9 @@ parameters: - name: TestPipeline type: boolean default: false - - name: Coverage - type: string - default: '' + - name: CoverageEnabled + type: boolean + default: true - name: CtestRegex type: string default: .* @@ -22,422 +22,174 @@ parameters: default: 'sdk/*/*/*cov_xml.xml' jobs: -- job: Validate - condition: and(succeededOrFailed(), ne(variables['Skip.Test'], 'true')) - strategy: - matrix: - Linux_x64_gcc5: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - # Can't set CC and CXX to env because it would affect VCPKG building which requires g++ > 7 - # So, this conf will set the CXX for cmake inline. - CmakeEnvArg: 'CC=/usr/bin/gcc-5 CXX=/usr/bin/g++-5 cmake' - BuildArgs: '-j 4' - Linux_x64_gcc8: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CC: '/usr/bin/gcc-8' - CXX: '/usr/bin/g++-8' - BuildArgs: '-j 10' - Linux_x64_gcc9: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CC: '/usr/bin/gcc-9' - CXX: '/usr/bin/g++-9' - BuildArgs: '-j 10' - Linux_x64_gcc9_no_rtti: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CC: '/usr/bin/gcc-9' - CXX: '/usr/bin/g++-9' - BuildArgs: '-j 10' - CmakeArgs: ' -DBUILD_RTTI=OFF' - Linux_x64: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - BuildArgs: '-j 10' - Win_x86: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' - VCPKG_DEFAULT_TRIPLET: 'x86-windows-static' - CMAKE_GENERATOR: 'Visual Studio 16 2019' - CMAKE_GENERATOR_PLATFORM: Win32 - CmakeArgs: ' -DBUILD_TRANSPORT_CURL=ON' #Leaving curl on here to explicitly test what the default behavior is on windows. - BuildArgs: '--parallel 8' - Win_x64: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' - VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' - CMAKE_GENERATOR: 'Visual Studio 16 2019' - CMAKE_GENERATOR_PLATFORM: x64 - BuildArgs: '--parallel 8' - Win_x64_no_rtti: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' - VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' - CMAKE_GENERATOR: 'Visual Studio 16 2019' - CMAKE_GENERATOR_PLATFORM: x64 - BuildArgs: '--parallel 8' - CmakeArgs: ' -DBUILD_RTTI=OFF' - Ubuntu_20_x64_clang: - Pool: azsdk-pool-mms-ubuntu-2004-general - OSVmImage: MMSUbuntu20.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CHECK_CLANG_FORMAT: 1 - AptDependencies: 'clang-11' - CC: '/usr/bin/clang-11' - CXX: '/usr/bin/clang++-11' - BuildArgs: '-j 4' - - # Unit testing ON - Linux_x64_with_unit_test: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CODE_COVERAGE=ON' - AptDependencies: 'gcovr lcov' - CODE_COVERAGE: '${{ parameters.Coverage }}' - # Make coverage report to avoid running the test exe because CI step will run it - CODE_COVERAGE_COLLECT_ONLY: 1 - BuildArgs: '-j 10' - Linux_x64_with_unit_test_release: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CmakeArgs: ' -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON' - BuildArgs: '-j 10' - Ubuntu20_x64_with_unit_test_release: - Pool: azsdk-pool-mms-ubuntu-2004-general - OSVmImage: MMSUbuntu20.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CmakeArgs: ' -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON' - BuildArgs: '-j 10' - Win_x86_with_unit_test: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' - VCPKG_DEFAULT_TRIPLET: 'x86-windows-static' - CMAKE_GENERATOR: 'Visual Studio 16 2019' - CMAKE_GENERATOR_PLATFORM: Win32 - CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_TRANSPORT_WINHTTP=ON' - BuildArgs: '--parallel 8' - Win_x64_with_unit_test: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' - VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' - CMAKE_GENERATOR: 'Visual Studio 16 2019' - CMAKE_GENERATOR_PLATFORM: x64 - CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_TRANSPORT_WINHTTP=ON' - BuildArgs: '--parallel 8' - MacOS_x64_with_unit_test: - Pool: Azure Pipelines - OSVmImage: 'macOS-10.14' - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-osx' - CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON' - BuildArgs: '-j 4' - pool: - name: $(Pool) - vmImage: $(OSVmImage) - - variables: - - name: CMOCKA_XML_FILE - value: "%g-test-results.xml" - - name: CMOCKA_MESSAGE_OUTPUT - value: "xml" - - name: BuildArgs - value: "" - - name: CmakeEnvArg - value: "" - - name: CmakeArgs - value: "" - - name: AZURE_TEST_MODE - value: "PLAYBACK" - - steps: - - checkout: self - submodules: recursive - - - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml + - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml parameters: - AgentImage: $(OSVmImage) - - # Add g++5 repo to ubuntu - - bash: sudo sh -c "echo 'deb http://ftp.debian.org/debian/ stretch main' >> /etc/apt/sources.list" - displayName: Add g++ 5 - condition: >- - and( - succeeded(), - contains(variables['OSVmImage'], 'Ubuntu'), - contains(variables['CmakeEnvArg'], 'CC=/usr/bin/gcc-5 CXX=/usr/bin/g++-5 cmake') - ) - - # Install g++5 - - bash: sudo apt-get update & sudo apt-get install g++-5 - displayName: Install g++ 5 - condition: >- - and( - succeeded(), - contains(variables['OSVmImage'], 'Ubuntu'), - contains(variables['CmakeEnvArg'], 'CC=/usr/bin/gcc-5 CXX=/usr/bin/g++-5 cmake') - ) - - # Install gcc and g++ 8 if it is needed on the image. - - bash: sudo apt-get install gcc-8 g++-8 -y - displayName: Install gcc and g++ 8 - condition: >- - and( - succeeded(), - contains(variables['OSVmImage'], 'Ubuntu'), - contains(variables['CC'], 'gcc-8'), - contains(variables['CXX'], 'g++-8') - ) - - # Validate all the files are formatted correctly according to the - # .clang-format file. This step runs on linux only only and assumes that - # clang-format-11 is installed. - - bash: | - # Run clang-format recursively on each source and header file within the repo sdk folder. - echo "Check clang-formatting" - clang-format --version - find ./sdk \( -iname '*.hpp' -o -iname '*.cpp' \) ! -iname 'json.hpp' -exec clang-format -i {} \; - - if [[ `git status | grep modified | awk '{print $2}'` ]]; then - echo Some files were not formatted correctly according to the .clang-format file. - echo Please run clang-format version 10 or greater to fix the issue by using this bash command at the root of the repo: - echo "find ./sdk \( -iname '*.hpp' -o -iname '*.cpp' \) ! -iname 'json.hpp' -exec clang-format -i {} \;" - echo "" - echo "List of files not formatted correctly:" - git status | grep modified | awk '{print $2}' - exit 1 - fi - - echo Success, all files are formatted correctly according to the .clang-format file. - exit 0 - - displayName: Validate Clang Format - condition: and(succeededOrFailed(), eq(variables['CHECK_CLANG_FORMAT'], 1)) - - - ${{ each artifact in parameters.Artifacts }}: - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageName: ${{ artifact.Name }} + MatrixConfigs: + - Name: base + Path: eng/pipelines/templates/stages/platform-matrix.json + Selection: all + GenerateVMJobs: true + JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml + AdditionalParameters: + Artifacts: ${{ parameters.Artifacts }} ServiceDirectory: ${{ parameters.ServiceDirectory }} TestPipeline: ${{ parameters.TestPipeline }} + CtestRegex: ${{ parameters.CtestRegex }} + CoverageReportPath: ${{ parameters.CoverageReportPath }} + CoverageEnabled: ${{ parameters.CoverageEnabled }} - # Install apt dependencies (if appropriate) - - bash: sudo apt install -y $(AptDependencies) - condition: and(succeededOrFailed(), ne(variables['AptDependencies'], '')) - displayName: Install dependencies from apt - - - template: /eng/pipelines/templates/steps/vcpkg.yml - parameters: - DependenciesVariableName: VcpkgInstall - - - script: | - dotnet tool install -g dotnet-reportgenerator-globaltool - dotnet tool install dotnet-reportgenerator-globaltool --tool-path tools - displayName: Install coverage tools - # CODE_COVERAGE variable is '' (do-not-generate) in all matrix but linux-gcc - # It is 'enabled' by default on linux-gcc but it can be opt-out by each pipeline (disabled) - condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], '')) - - - template: /eng/pipelines/templates/steps/cmake-build.yml - parameters: - GenerateArgs: "$(CmakeArgs)" - BuildArgs: "$(BuildArgs)" - Env: "$(CmakeEnvArg)" - - - script: ctest -C Debug -V --tests-regex ${{ parameters.CtestRegex }} - workingDirectory: build - displayName: ctest - - # Make coverage targets (specified in coverage_targets.txt) and assemble - # coverage report - - bash: | - make `cat ${{ parameters.ServiceDirectory }}-targets-coverage.txt` - ../tools/reportgenerator "-reports:${{ parameters.CoverageReportPath }}" "-targetdir:." "-reporttypes:Cobertura" - workingDirectory: build - displayName: Generate Code Coverage Data - condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], '')) - - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(Build.SourcesDirectory)/**/Cobertura.xml' - displayName: Publish Code Coverage to DevOps - condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], '')) - -# Disable build for cpp - client -- ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}: - - job: GenerateReleaseArtifacts - pool: - name: azsdk-pool-mms-win-2019-general - vmImage: MMS2019 - variables: - VcpkgDependencies: curl[winssl] libxml2 - VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' - steps: - - template: /eng/common/pipelines/templates/steps/check-spelling.yml - parameters: - ContinueOnError: false - - - template: /eng/common/pipelines/templates/steps/verify-links.yml - parameters: - Directory: '' - CheckLinkGuidance: $true - - - ${{ each artifact in parameters.Artifacts }}: - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + # Disable build for cpp - client + - ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}: + - job: GenerateReleaseArtifacts + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + variables: + VcpkgDependencies: curl[winssl] libxml2 + VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' + steps: + - template: /eng/common/pipelines/templates/steps/check-spelling.yml parameters: - PackageName: ${{ artifact.Name }} - ServiceDirectory: ${{ parameters.ServiceDirectory }} - TestPipeline: ${{ parameters.TestPipeline }} + ContinueOnError: false - - pwsh: | - $setupFile="doxygen-1.8.18-setup.exe"; + - template: /eng/common/pipelines/templates/steps/verify-links.yml + parameters: + Directory: '' + CheckLinkGuidance: $true - Invoke-WebRequest ` - -MaximumRetryCount 10 ` - -Uri "https://azuresdktooling.blob.core.windows.net/dependencies/$setupFile" ` - -OutFile $setupFile ` - | Wait-Process; + - ${{ each artifact in parameters.Artifacts }}: + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: ${{ artifact.Name }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestPipeline: ${{ parameters.TestPipeline }} - Start-Process -Filepath .\$setupFile -ArgumentList @("/VERYSILENT") -Wait - workingDirectory: $(Agent.TempDirectory) - displayName: Download and Install Doxygen + - pwsh: | + $setupFile="doxygen-1.8.18-setup.exe"; - # Install dependencies required for build from vcpkg - # TODO: We should be able to generate docs without having to install these - # prerequisites: - - template: /eng/pipelines/templates/steps/vcpkg.yml - parameters: - DependenciesVariableName: VcpkgDependencies + Invoke-WebRequest ` + -MaximumRetryCount 10 ` + -Uri "https://azuresdktooling.blob.core.windows.net/dependencies/$setupFile" ` + -OutFile $setupFile ` + | Wait-Process; - - template: /eng/pipelines/templates/steps/cmake-build.yml - parameters: - Build: false - GenerateArgs: >- - -DINSTALL_GTEST=OFF - -DBUILD_TESTING=OFF - -DBUILD_TRANSPORT_CURL=OFF - -DBUILD_DOCUMENTATION=YES + Start-Process -Filepath .\$setupFile -ArgumentList @("/VERYSILENT") -Wait + workingDirectory: $(Agent.TempDirectory) + displayName: Download and Install Doxygen - - pwsh: npm install -g moxygen - displayName: Install Moxygen to generate markdown for docs.microsoft.com + # Install dependencies required for build from vcpkg + # TODO: We should be able to generate docs without having to install these + # prerequisites: + - template: /eng/pipelines/templates/steps/vcpkg.yml + parameters: + DependenciesVariableName: VcpkgDependencies - # Generate package spec JSON files for tagging the repo - - ${{ each artifact in parameters.Artifacts }}: - - pwsh: >- - New-Item - -ItemType directory - -Path $(Build.ArtifactStagingDirectory) - -Name packages/${{ artifact.Name }} - displayName: Create package info artifact location + - template: /eng/pipelines/templates/steps/cmake-build.yml + parameters: + Build: false + GenerateArgs: >- + -DINSTALL_GTEST=OFF + -DBUILD_TESTING=OFF + -DBUILD_TRANSPORT_CURL=OFF + -DBUILD_DOCUMENTATION=YES - - pwsh: | - $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }}/package-info.json + - pwsh: npm install -g moxygen + displayName: Install Moxygen to generate markdown for docs.microsoft.com - $version = eng/scripts/Get-PkgVersion -ServiceDirectory ${{ parameters.ServiceDirectory }} -PackageName ${{ artifact.Path }} - $outputObject = @{ version = $version.ToString(); name = '${{ artifact.Name }}'; packageName = "${{ artifact.Name }}_$version"; } | ConvertTo-Json + # Generate package spec JSON files for tagging the repo + - ${{ each artifact in parameters.Artifacts }}: + - pwsh: >- + New-Item + -ItemType directory + -Path $(Build.ArtifactStagingDirectory) + -Name packages/${{ artifact.Name }} + displayName: Create package info artifact location - Set-Content -Path $outputPath -Value $outputObject + - pwsh: | + $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }}/package-info.json - # Set AZURE_SDK_VERSION for use in Doxygen generation - Write-Host "##vso[task.setvariable variable=AZURE_SDK_VERSION]$version" - displayName: Create package info JSON file + $version = eng/scripts/Get-PkgVersion -ServiceDirectory ${{ parameters.ServiceDirectory }} -PackageName ${{ artifact.Path }} + $outputObject = @{ version = $version.ToString(); name = '${{ artifact.Name }}'; packageName = "${{ artifact.Name }}_$version"; } | ConvertTo-Json - - task: Powershell@2 - inputs: - targetType: inline - script: | - # If there is no vcpkg directory skip staging the vcpkg port artifacts - if (!(Test-Path build/vcpkg/ports/${{ artifact.VcpkgPortName }})) { - Write-Host "No vcpkg port directory at build/vcpkg/ports/${{ artifact.VcpkgPortName }}" - exit + Set-Content -Path $outputPath -Value $outputObject + + # Set AZURE_SDK_VERSION for use in Doxygen generation + Write-Host "##vso[task.setvariable variable=AZURE_SDK_VERSION]$version" + displayName: Create package info JSON file + + - task: Powershell@2 + inputs: + targetType: inline + script: | + # If there is no vcpkg directory skip staging the vcpkg port artifacts + if (!(Test-Path build/vcpkg/ports/${{ artifact.VcpkgPortName }})) { + Write-Host "No vcpkg port directory at build/vcpkg/ports/${{ artifact.VcpkgPortName }}" + exit + } + + New-Item ` + -ItemType Directory ` + $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg -Force + + Copy-Item -Recurse ` + build/vcpkg/ports/${{ artifact.VcpkgPortName }}/. ` + $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg/port + pwsh: true + displayName: Copy vcpkg port files from build + + - pwsh: | + $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }} + $changelogPath = "sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md" + + if (Test-Path $changeLogPath) { + Copy-Item sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md $outputPath + } else { + Write-Warning "$changeLogPath does not exist" } + displayName: Copy CHANGELOG.md to package artifact - New-Item ` - -ItemType Directory ` - $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg -Force + - script: cmake --build . --target ${{ artifact.Name }}-docs + workingDirectory: build + displayName: Generate docs (${{ artifact.Name }}-docs) - Copy-Item -Recurse ` - build/vcpkg/ports/${{ artifact.VcpkgPortName }}/. ` - $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg/port - pwsh: true - displayName: Copy vcpkg port files from build + - task: PowerShell@2 + inputs: + targetType: filePath + filePath: eng/scripts/New-DocsMsArtifact.ps1 + arguments: >- + -ServiceDirectory ${{ parameters.ServiceDirectory }} + -PackageName ${{ artifact.Name }} + -TargetFolder $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/docs/docs.ms + ignoreLASTEXITCODE: true + pwsh: true + displayName: Generate artifacts for docs.ms - - pwsh: | - $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }} - $changelogPath = "sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md" + - pwsh: | + New-Item -ItemType directory -Path $(Build.ArtifactStagingDirectory) -Name docs/${{ artifact.Name }} + displayName: Create doc artifact location - if (Test-Path $changeLogPath) { - Copy-Item sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md $outputPath - } else { - Write-Warning "$changeLogPath does not exist" - } - displayName: Copy CHANGELOG.md to package artifact + - task: CopyFiles@2 + inputs: + sourceFolder: $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }} + contents: package-info.json + targetFolder: build/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/docs/html + displayName: Copy package-info.json to documentation path - - script: cmake --build . --target ${{ artifact.Name }}-docs - workingDirectory: build - displayName: Generate docs (${{ artifact.Name }}-docs) + - task: CopyFiles@2 + inputs: + sourceFolder: build/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/docs/html + targetFolder: $(Build.ArtifactStagingDirectory)/docs/${{ artifact.Name }} + displayName: Copy documentation to artifact staging directory - - task: PowerShell@2 - inputs: - targetType: filePath - filePath: eng/scripts/New-DocsMsArtifact.ps1 - arguments: >- - -ServiceDirectory ${{ parameters.ServiceDirectory }} - -PackageName ${{ artifact.Name }} - -TargetFolder $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/docs/docs.ms - ignoreLASTEXITCODE: true - pwsh: true - displayName: Generate artifacts for docs.ms + - task: PublishPipelineArtifact@1 + inputs: + artifactName: packages + path: $(Build.ArtifactStagingDirectory)/packages - - pwsh: | - New-Item -ItemType directory -Path $(Build.ArtifactStagingDirectory) -Name docs/${{ artifact.Name }} - displayName: Create doc artifact location + # After all docs artifacts are generated publish docs artifacts + - task: PublishPipelineArtifact@1 + inputs: + artifactName: docs + path: $(Build.ArtifactStagingDirectory)/docs - - task: CopyFiles@2 - inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }} - contents: package-info.json - targetFolder: build/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/docs/html - displayName: Copy package-info.json to documentation path - - - task: CopyFiles@2 - inputs: - sourceFolder: build/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/docs/html - targetFolder: $(Build.ArtifactStagingDirectory)/docs/${{ artifact.Name }} - displayName: Copy documentation to artifact staging directory - - - task: PublishPipelineArtifact@1 - inputs: - artifactName: packages - path: $(Build.ArtifactStagingDirectory)/packages - - # After all docs artifacts are generated publish docs artifacts - - task: PublishPipelineArtifact@1 - inputs: - artifactName: docs - path: $(Build.ArtifactStagingDirectory)/docs - - - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml + - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml diff --git a/eng/pipelines/templates/jobs/archetype-sdk-tests.yml b/eng/pipelines/templates/jobs/archetype-sdk-tests.yml index a9a97a243..d236f3cc3 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-tests.yml @@ -21,6 +21,11 @@ parameters: type: number default: 60 +# This job uses the legacy matrix format (matrix property of the job, one +# build/test scenario per job). A new format (multiple build/test scenarios per +# job) may be possible here: +# https://github.com/Azure/azure-sdk-for-cpp/issues/2944 + jobs: - job: Validate condition: and(succeededOrFailed(), ne(variables['Skip.LiveTest'], 'true')) @@ -68,7 +73,6 @@ jobs: Win_x86_with_unit_test_winHttp: Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 - VcpkgInstall: 'libxml2' VCPKG_DEFAULT_TRIPLET: 'x86-windows-static' CMAKE_GENERATOR: 'Visual Studio 16 2019' CMAKE_GENERATOR_PLATFORM: Win32 @@ -87,7 +91,6 @@ jobs: Win_x64_with_unit_test_winHttp: Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 - VcpkgInstall: 'libxml2' VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' CMAKE_GENERATOR: 'Visual Studio 16 2019' CMAKE_GENERATOR_PLATFORM: x64 @@ -98,7 +101,7 @@ jobs: Win_x86_with_unit_test_libcurl: Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' + VcpkgInstall: 'curl[winssl]' VCPKG_DEFAULT_TRIPLET: 'x86-windows-static' CMAKE_GENERATOR: 'Visual Studio 16 2019' CMAKE_GENERATOR_PLATFORM: Win32 @@ -108,7 +111,7 @@ jobs: Win_x64_with_unit_test_libcurl: Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' + VcpkgInstall: 'curl[winssl]' VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' CMAKE_GENERATOR: 'Visual Studio 16 2019' CMAKE_GENERATOR_PLATFORM: x64 diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml new file mode 100644 index 000000000..fc3c0fcc4 --- /dev/null +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -0,0 +1,138 @@ +parameters: + - name: Artifacts + type: object + default: [] + - name: ServiceDirectory + type: string + default: not-specified + - name: TestPipeline + type: boolean + default: false + - name: CtestRegex + type: string + default: .* + - name: CoverageReportPath + type: string + default: 'sdk/*/*/*cov_xml.xml' + - name: CoverageEnabled + type: boolean + default: true + - name: Matrix + type: string + + # Supplied by archetype-sdk-tests-generate.yml dynamic matrix generator + - name: CloudConfig + type: object + default: {} + # Supplied by archetype-sdk-tests-generate.yml dynamic matrix generator. + # Must be wired up to ensure population of parameters.Matrix + - name: DependsOn + type: string + default: '' + # Supplied by archetype-sdk-tests-generate.yml dynamic matrix generator + - name: UsePlatformContainer + type: boolean + default: false + +jobs: + - job: + displayName: "Validate" + dependsOn: ${{ parameters.DependsOn }} + condition: and(succeededOrFailed(), ne(variables['Skip.Test'], 'true'), ne(${{ parameters.Matrix }}, '{}')) + strategy: + matrix: $[ ${{ parameters.Matrix }} ] + maxParallel: 12 + pool: + vmImage: $(OSVmImage) + name: $(Pool) + variables: + - name: CMOCKA_XML_FILE + value: "%g-test-results.xml" + - name: CMOCKA_MESSAGE_OUTPUT + value: "xml" + - name: BuildArgs + value: "" + - name: CmakeEnvArg + value: "" + - name: CmakeArgs + value: "" + - name: AZURE_TEST_MODE + value: "PLAYBACK" + steps: + - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml + parameters: + AgentImage: $(OsVmImage) + + - pwsh: sudo apt update && sudo apt install -y $(AptDependencies) + condition: and(succeeded(), ne(variables['AptDependencies'], '')) + displayName: Install dependencies from apt + + - template: /eng/pipelines/templates/steps/vcpkg.yml + parameters: + DependenciesVariableName: VcpkgInstall + + # Validate all the files are formatted correctly according to the + # .clang-format file. This step runs on linux only only and assumes that + # clang-format-11 is installed. + - bash: | + # Run clang-format recursively on each source and header file within the repo sdk folder. + echo "Check clang-formatting" + clang-format --version + find ./sdk \( -iname '*.hpp' -o -iname '*.cpp' \) ! -iname 'json.hpp' -exec clang-format -i {} \; + + if [[ `git status | grep modified | awk '{print $2}'` ]]; then + echo Some files were not formatted correctly according to the .clang-format file. + echo Please run clang-format version 10 or greater to fix the issue by using this bash command at the root of the repo: + echo "find ./sdk \( -iname '*.hpp' -o -iname '*.cpp' \) ! -iname 'json.hpp' -exec clang-format -i {} \;" + echo "" + echo "List of files not formatted correctly:" + git status | grep modified | awk '{print $2}' + exit 1 + fi + + echo Success, all files are formatted correctly according to the .clang-format file. + exit 0 + + displayName: Validate Clang Format + condition: and(succeededOrFailed(), eq(variables['CHECK_CLANG_FORMAT'], 1)) + + - ${{ each artifact in parameters.Artifacts }}: + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: ${{ artifact.Name }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestPipeline: ${{ parameters.TestPipeline }} + + - template: /eng/pipelines/templates/steps/cmake-build.yml + parameters: + GenerateArgs: "$(CmakeArgs)" + BuildArgs: "$(BuildArgs)" + Env: "$(CmakeEnvArg)" + + - script: ctest -C Debug -V --tests-regex ${{ parameters.CtestRegex }} + workingDirectory: build + + - ${{ if eq(parameters.CoverageEnabled, true) }}: + - pwsh: | + $toolsDirectory = "$(Agent.TempDirectory)/coveragetools" + dotnet tool install -g dotnet-reportgenerator-globaltool + dotnet tool install dotnet-reportgenerator-globaltool --tool-path $toolsDirectory + Write-Host "##vso[task.setvariable variable=ToolsDirectory]$toolsDirectory" + displayName: Install coverage tools + condition: and(succeeded(), eq(variables['CODE_COVERAGE'], 'enabled')) + + # Make coverage targets (specified in coverage_targets.txt) and assemble + # coverage report + - bash: | + make VERBOSE=1 `cat ${{ parameters.ServiceDirectory }}-targets-coverage.txt` + $(ToolsDirectory)/reportgenerator "-reports:${{ parameters.CoverageReportPath }}" "-targetdir:." "-reporttypes:Cobertura" + workingDirectory: build + displayName: Generate Code Coverage Data + condition: and(succeededOrFailed(), eq(variables['CODE_COVERAGE'], 'enabled')) + + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(Build.SourcesDirectory)/**/Cobertura.xml' + displayName: Publish Code Coverage to DevOps + condition: and(succeededOrFailed(), eq(variables['CODE_COVERAGE'], 'enabled')) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 14aa8ce92..0c9eb5831 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -5,9 +5,9 @@ parameters: - name: CtestRegex type: string default: .* -- name: Coverage - type: string - default: 'enabled' +- name: CoverageEnabled + type: boolean + default: true - name: LiveTestCtestRegex type: string default: '' @@ -35,7 +35,7 @@ stages: ServiceDirectory: ${{ parameters.ServiceDirectory }} Artifacts: ${{ parameters.Artifacts }} CtestRegex: ${{ parameters.CtestRegex }} - Coverage: ${{ parameters.Coverage }} + CoverageEnabled: ${{ parameters.CoverageEnabled }} CoverageReportPath: ${{ parameters.CoverageReportPath }} ${{ if eq(parameters.ServiceDirectory, 'template') }}: TestPipeline: true diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json new file mode 100644 index 000000000..f58ee8dd5 --- /dev/null +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -0,0 +1,154 @@ +{ + "displayNames": { + "_": "" + }, + "include": [ + { + "OSVmImage": ["macOS-10.14", "macOS-10.15", "macOS-11"], + "StaticConfigs": { + "_": { + "Pool": "Azure Pipelines", + "VcpkgInstall": "curl[ssl] libxml2 openssl", + "BuildArgs": "-j 10", + "VCPKG_DEFAULT_TRIPLET": "x64-osx", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON" + } + }, + "BuildConfig": { + "debug": { + "CMAKE_BUILD_TYPE": "Debug" + }, + "release": { + "CMAKE_BUILD_TYPE": "Release" + } + } + }, + { + "StaticConfigs": { + "Windows2019": { + "OSVmImage": "MMS2019", + "Pool": "azsdk-pool-mms-win-2019-general", + "BuildArgs": "--parallel 8", + "CMAKE_GENERATOR": "Visual Studio 16 2019" + } + }, + "TargetPlatform": { + "Win32Api_curl": { + "VcpkgInstall": "curl[winssl]", + "CmakeArgs": " -DBUILD_TRANSPORT_CURL=ON" + }, + "Win32Api_release_curl": { + "VcpkgInstall": "curl[winssl]", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON", + "CMAKE_BUILD_TYPE": "Release" + }, + "Win32Api_debug_tests": { + "VcpkgInstall": "curl[winssl]", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_TRANSPORT_WINHTTP=ON" + } + }, + "TargetArchitecture": { + "x86": { + "CMAKE_GENERATOR_PLATFORM": "Win32", + "VCPKG_DEFAULT_TRIPLET": "x86-windows-static" + }, + "x64": { + "CMAKE_GENERATOR_PLATFORM": "x64", + "VCPKG_DEFAULT_TRIPLET": "x64-windows-static" + } + } + }, + { + "StaticConfigs": { + "Windows2019": { + "OSVmImage": "MMS2019", + "Pool": "azsdk-pool-mms-win-2019-general", + "BuildArgs": "--parallel 8", + "CMAKE_GENERATOR": "Visual Studio 16 2019", + "CmakeArgs": " -DBUILD_TRANSPORT_WINHTTP=ON -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON" + } + }, + "TargetPlatform": { + "UWP_debug": { + "CMAKE_SYSTEM_NAME": "WindowsStore", + "CMAKE_SYSTEM_VERSION": "10.0" + }, + "UWP_release": { + "CMAKE_SYSTEM_NAME": "WindowsStore", + "CMAKE_SYSTEM_VERSION": "10.0", + "CMAKE_BUILD_TYPE": "Release" + } + }, + "TargetArchitecture": { + "x86": { + "CMAKE_GENERATOR_PLATFORM": "Win32" + }, + "x64": { + "CMAKE_GENERATOR_PLATFORM": "x64", + "VCPKG_DEFAULT_TRIPLET": "x64-uwp" + } + } + }, + { + "StaticConfigs": { + "Ubuntu18": { + "OSVmImage": "MMSUbuntu18.04", + "Pool": "azsdk-pool-mms-ubuntu-1804-general", + "VcpkgInstall": "curl[ssl] libxml2 openssl", + "VCPKG_DEFAULT_TRIPLET": "x64-linux", + "BuildArgs": "-j 10" + } + }, + "BuildSettings": { + "gpp-5": { + "AptDependencies": "g++-5", + "CmakeEnvArg": "CC=/usr/bin/gcc-5 CXX=/usr/bin/g++-5 " + }, + "gpp-8": { + "AptDependencies": "g++-8", + "CC": "/usr/bin/gcc-8", + "CXX": "/usr/bin/g++-8" + }, + "gpp-9": { + "AptDependencies": "g++-9", + "CC": "/usr/bin/gcc-9", + "CXX": "/usr/bin/g++-9" + }, + "included_coverage": { + "AptDependencies": "gcovr lcov", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CODE_COVERAGE=ON -DCMAKE_VERBOSE_MAKEFILE=ON", + "CODE_COVERAGE_COLLECT_ONLY": "1", + "CODE_COVERAGE": "enabled" + }, + "included": { }, + "included_release": { + "CMAKE_BUILD_TYPE": "Release", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON" + } + } + }, + { + "StaticConfigs": { + "Ubuntu20": { + "OSVmImage": "MMSUbuntu20.04", + "Pool": "azsdk-pool-mms-ubuntu-2004-general", + "VcpkgInstall": "curl[ssl] libxml2 openssl", + "VCPKG_DEFAULT_TRIPLET": "x64-linux", + "BuildArgs": "-j 10", + "CC": "/usr/bin/clang-11", + "CXX": "/usr/bin/clang++-11", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON" + } + }, + "BuildSettings": { + "clang-11": { + "CHECK_CLANG_FORMAT": "1" + }, + "included_release": { + "CMAKE_BUILD_TYPE": "Release" + + } + } + } + ] +} diff --git a/eng/pipelines/templates/steps/vcpkg.yml b/eng/pipelines/templates/steps/vcpkg.yml index 9eee8179f..94097e562 100644 --- a/eng/pipelines/templates/steps/vcpkg.yml +++ b/eng/pipelines/templates/steps/vcpkg.yml @@ -25,7 +25,7 @@ steps: ) # This task times out after 10 minutes. An analysis of a subset of Vcpkg Cache - # runs showed that after 10 minutes the cache step is very unlikely to + # runs showed that after 10 minutes the cache step is very unlikely to # succeed. - task: Cache@2 inputs: @@ -33,7 +33,6 @@ steps: $(Agent.JobName) | "$(VcpkgCommit)" | $(Agent.Os) - | $(${{ parameters.DependenciesVariableName }}) path: $(VCPKG_INSTALLATION_ROOT) cacheHitVar: VcpkgRestoredFromCache displayName: Vcpkg Cache diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index b931d800f..5835f5873 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -37,7 +37,7 @@ stages: ServiceDirectory: storage # Not running tests on CI but only on LiveTest mode CtestRegex: no-run - Coverage: disabled + CoverageEnabled: false LiveTestCtestRegex: azure-storage SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview) Artifacts: