diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 4501c1f8c..476b29d62 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -113,7 +113,7 @@ ], "overrides": [ { - "filename": "**/eng/pipelines/templates/jobs/archetype-sdk-client.yml", + "filename": "**/eng/pipelines/templates/**/*.yml", "words": [ "lcov", "iname" diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 2794568b4..5dfa64140 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -8,9 +8,9 @@ parameters: - name: TestPipeline type: boolean default: false - - name: Coverage - type: string - default: '' + - name: CoverageEnabled + type: boolean + default: true - name: CtestRegex type: string default: .* @@ -22,422 +22,174 @@ parameters: default: 'sdk/*/*/*cov_xml.xml' jobs: -- job: Validate - condition: and(succeededOrFailed(), ne(variables['Skip.Test'], 'true')) - strategy: - matrix: - Linux_x64_gcc5: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - # Can't set CC and CXX to env because it would affect VCPKG building which requires g++ > 7 - # So, this conf will set the CXX for cmake inline. - CmakeEnvArg: 'CC=/usr/bin/gcc-5 CXX=/usr/bin/g++-5 cmake' - BuildArgs: '-j 4' - Linux_x64_gcc8: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CC: '/usr/bin/gcc-8' - CXX: '/usr/bin/g++-8' - BuildArgs: '-j 10' - Linux_x64_gcc9: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CC: '/usr/bin/gcc-9' - CXX: '/usr/bin/g++-9' - BuildArgs: '-j 10' - Linux_x64_gcc9_no_rtti: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CC: '/usr/bin/gcc-9' - CXX: '/usr/bin/g++-9' - BuildArgs: '-j 10' - CmakeArgs: ' -DBUILD_RTTI=OFF' - Linux_x64: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - BuildArgs: '-j 10' - Win_x86: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' - VCPKG_DEFAULT_TRIPLET: 'x86-windows-static' - CMAKE_GENERATOR: 'Visual Studio 16 2019' - CMAKE_GENERATOR_PLATFORM: Win32 - CmakeArgs: ' -DBUILD_TRANSPORT_CURL=ON' #Leaving curl on here to explicitly test what the default behavior is on windows. - BuildArgs: '--parallel 8' - Win_x64: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' - VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' - CMAKE_GENERATOR: 'Visual Studio 16 2019' - CMAKE_GENERATOR_PLATFORM: x64 - BuildArgs: '--parallel 8' - Win_x64_no_rtti: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' - VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' - CMAKE_GENERATOR: 'Visual Studio 16 2019' - CMAKE_GENERATOR_PLATFORM: x64 - BuildArgs: '--parallel 8' - CmakeArgs: ' -DBUILD_RTTI=OFF' - Ubuntu_20_x64_clang: - Pool: azsdk-pool-mms-ubuntu-2004-general - OSVmImage: MMSUbuntu20.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CHECK_CLANG_FORMAT: 1 - AptDependencies: 'clang-11' - CC: '/usr/bin/clang-11' - CXX: '/usr/bin/clang++-11' - BuildArgs: '-j 4' - - # Unit testing ON - Linux_x64_with_unit_test: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CODE_COVERAGE=ON' - AptDependencies: 'gcovr lcov' - CODE_COVERAGE: '${{ parameters.Coverage }}' - # Make coverage report to avoid running the test exe because CI step will run it - CODE_COVERAGE_COLLECT_ONLY: 1 - BuildArgs: '-j 10' - Linux_x64_with_unit_test_release: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CmakeArgs: ' -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON' - BuildArgs: '-j 10' - Ubuntu20_x64_with_unit_test_release: - Pool: azsdk-pool-mms-ubuntu-2004-general - OSVmImage: MMSUbuntu20.04 - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CmakeArgs: ' -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON' - BuildArgs: '-j 10' - Win_x86_with_unit_test: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' - VCPKG_DEFAULT_TRIPLET: 'x86-windows-static' - CMAKE_GENERATOR: 'Visual Studio 16 2019' - CMAKE_GENERATOR_PLATFORM: Win32 - CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_TRANSPORT_WINHTTP=ON' - BuildArgs: '--parallel 8' - Win_x64_with_unit_test: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' - VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' - CMAKE_GENERATOR: 'Visual Studio 16 2019' - CMAKE_GENERATOR_PLATFORM: x64 - CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_TRANSPORT_WINHTTP=ON' - BuildArgs: '--parallel 8' - MacOS_x64_with_unit_test: - Pool: Azure Pipelines - OSVmImage: 'macOS-10.14' - VcpkgInstall: 'curl[ssl] libxml2 openssl' - VCPKG_DEFAULT_TRIPLET: 'x64-osx' - CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON' - BuildArgs: '-j 4' - pool: - name: $(Pool) - vmImage: $(OSVmImage) - - variables: - - name: CMOCKA_XML_FILE - value: "%g-test-results.xml" - - name: CMOCKA_MESSAGE_OUTPUT - value: "xml" - - name: BuildArgs - value: "" - - name: CmakeEnvArg - value: "" - - name: CmakeArgs - value: "" - - name: AZURE_TEST_MODE - value: "PLAYBACK" - - steps: - - checkout: self - submodules: recursive - - - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml + - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml parameters: - AgentImage: $(OSVmImage) - - # Add g++5 repo to ubuntu - - bash: sudo sh -c "echo 'deb http://ftp.debian.org/debian/ stretch main' >> /etc/apt/sources.list" - displayName: Add g++ 5 - condition: >- - and( - succeeded(), - contains(variables['OSVmImage'], 'Ubuntu'), - contains(variables['CmakeEnvArg'], 'CC=/usr/bin/gcc-5 CXX=/usr/bin/g++-5 cmake') - ) - - # Install g++5 - - bash: sudo apt-get update & sudo apt-get install g++-5 - displayName: Install g++ 5 - condition: >- - and( - succeeded(), - contains(variables['OSVmImage'], 'Ubuntu'), - contains(variables['CmakeEnvArg'], 'CC=/usr/bin/gcc-5 CXX=/usr/bin/g++-5 cmake') - ) - - # Install gcc and g++ 8 if it is needed on the image. - - bash: sudo apt-get install gcc-8 g++-8 -y - displayName: Install gcc and g++ 8 - condition: >- - and( - succeeded(), - contains(variables['OSVmImage'], 'Ubuntu'), - contains(variables['CC'], 'gcc-8'), - contains(variables['CXX'], 'g++-8') - ) - - # Validate all the files are formatted correctly according to the - # .clang-format file. This step runs on linux only only and assumes that - # clang-format-11 is installed. - - bash: | - # Run clang-format recursively on each source and header file within the repo sdk folder. - echo "Check clang-formatting" - clang-format --version - find ./sdk \( -iname '*.hpp' -o -iname '*.cpp' \) ! -iname 'json.hpp' -exec clang-format -i {} \; - - if [[ `git status | grep modified | awk '{print $2}'` ]]; then - echo Some files were not formatted correctly according to the .clang-format file. - echo Please run clang-format version 10 or greater to fix the issue by using this bash command at the root of the repo: - echo "find ./sdk \( -iname '*.hpp' -o -iname '*.cpp' \) ! -iname 'json.hpp' -exec clang-format -i {} \;" - echo "" - echo "List of files not formatted correctly:" - git status | grep modified | awk '{print $2}' - exit 1 - fi - - echo Success, all files are formatted correctly according to the .clang-format file. - exit 0 - - displayName: Validate Clang Format - condition: and(succeededOrFailed(), eq(variables['CHECK_CLANG_FORMAT'], 1)) - - - ${{ each artifact in parameters.Artifacts }}: - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageName: ${{ artifact.Name }} + MatrixConfigs: + - Name: base + Path: eng/pipelines/templates/stages/platform-matrix.json + Selection: all + GenerateVMJobs: true + JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml + AdditionalParameters: + Artifacts: ${{ parameters.Artifacts }} ServiceDirectory: ${{ parameters.ServiceDirectory }} TestPipeline: ${{ parameters.TestPipeline }} + CtestRegex: ${{ parameters.CtestRegex }} + CoverageReportPath: ${{ parameters.CoverageReportPath }} + CoverageEnabled: ${{ parameters.CoverageEnabled }} - # Install apt dependencies (if appropriate) - - bash: sudo apt install -y $(AptDependencies) - condition: and(succeededOrFailed(), ne(variables['AptDependencies'], '')) - displayName: Install dependencies from apt - - - template: /eng/pipelines/templates/steps/vcpkg.yml - parameters: - DependenciesVariableName: VcpkgInstall - - - script: | - dotnet tool install -g dotnet-reportgenerator-globaltool - dotnet tool install dotnet-reportgenerator-globaltool --tool-path tools - displayName: Install coverage tools - # CODE_COVERAGE variable is '' (do-not-generate) in all matrix but linux-gcc - # It is 'enabled' by default on linux-gcc but it can be opt-out by each pipeline (disabled) - condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], '')) - - - template: /eng/pipelines/templates/steps/cmake-build.yml - parameters: - GenerateArgs: "$(CmakeArgs)" - BuildArgs: "$(BuildArgs)" - Env: "$(CmakeEnvArg)" - - - script: ctest -C Debug -V --tests-regex ${{ parameters.CtestRegex }} - workingDirectory: build - displayName: ctest - - # Make coverage targets (specified in coverage_targets.txt) and assemble - # coverage report - - bash: | - make `cat ${{ parameters.ServiceDirectory }}-targets-coverage.txt` - ../tools/reportgenerator "-reports:${{ parameters.CoverageReportPath }}" "-targetdir:." "-reporttypes:Cobertura" - workingDirectory: build - displayName: Generate Code Coverage Data - condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], '')) - - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(Build.SourcesDirectory)/**/Cobertura.xml' - displayName: Publish Code Coverage to DevOps - condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], '')) - -# Disable build for cpp - client -- ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}: - - job: GenerateReleaseArtifacts - pool: - name: azsdk-pool-mms-win-2019-general - vmImage: MMS2019 - variables: - VcpkgDependencies: curl[winssl] libxml2 - VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' - steps: - - template: /eng/common/pipelines/templates/steps/check-spelling.yml - parameters: - ContinueOnError: false - - - template: /eng/common/pipelines/templates/steps/verify-links.yml - parameters: - Directory: '' - CheckLinkGuidance: $true - - - ${{ each artifact in parameters.Artifacts }}: - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + # Disable build for cpp - client + - ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}: + - job: GenerateReleaseArtifacts + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + variables: + VcpkgDependencies: curl[winssl] libxml2 + VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' + steps: + - template: /eng/common/pipelines/templates/steps/check-spelling.yml parameters: - PackageName: ${{ artifact.Name }} - ServiceDirectory: ${{ parameters.ServiceDirectory }} - TestPipeline: ${{ parameters.TestPipeline }} + ContinueOnError: false - - pwsh: | - $setupFile="doxygen-1.8.18-setup.exe"; + - template: /eng/common/pipelines/templates/steps/verify-links.yml + parameters: + Directory: '' + CheckLinkGuidance: $true - Invoke-WebRequest ` - -MaximumRetryCount 10 ` - -Uri "https://azuresdktooling.blob.core.windows.net/dependencies/$setupFile" ` - -OutFile $setupFile ` - | Wait-Process; + - ${{ each artifact in parameters.Artifacts }}: + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: ${{ artifact.Name }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestPipeline: ${{ parameters.TestPipeline }} - Start-Process -Filepath .\$setupFile -ArgumentList @("/VERYSILENT") -Wait - workingDirectory: $(Agent.TempDirectory) - displayName: Download and Install Doxygen + - pwsh: | + $setupFile="doxygen-1.8.18-setup.exe"; - # Install dependencies required for build from vcpkg - # TODO: We should be able to generate docs without having to install these - # prerequisites: - - template: /eng/pipelines/templates/steps/vcpkg.yml - parameters: - DependenciesVariableName: VcpkgDependencies + Invoke-WebRequest ` + -MaximumRetryCount 10 ` + -Uri "https://azuresdktooling.blob.core.windows.net/dependencies/$setupFile" ` + -OutFile $setupFile ` + | Wait-Process; - - template: /eng/pipelines/templates/steps/cmake-build.yml - parameters: - Build: false - GenerateArgs: >- - -DINSTALL_GTEST=OFF - -DBUILD_TESTING=OFF - -DBUILD_TRANSPORT_CURL=OFF - -DBUILD_DOCUMENTATION=YES + Start-Process -Filepath .\$setupFile -ArgumentList @("/VERYSILENT") -Wait + workingDirectory: $(Agent.TempDirectory) + displayName: Download and Install Doxygen - - pwsh: npm install -g moxygen - displayName: Install Moxygen to generate markdown for docs.microsoft.com + # Install dependencies required for build from vcpkg + # TODO: We should be able to generate docs without having to install these + # prerequisites: + - template: /eng/pipelines/templates/steps/vcpkg.yml + parameters: + DependenciesVariableName: VcpkgDependencies - # Generate package spec JSON files for tagging the repo - - ${{ each artifact in parameters.Artifacts }}: - - pwsh: >- - New-Item - -ItemType directory - -Path $(Build.ArtifactStagingDirectory) - -Name packages/${{ artifact.Name }} - displayName: Create package info artifact location + - template: /eng/pipelines/templates/steps/cmake-build.yml + parameters: + Build: false + GenerateArgs: >- + -DINSTALL_GTEST=OFF + -DBUILD_TESTING=OFF + -DBUILD_TRANSPORT_CURL=OFF + -DBUILD_DOCUMENTATION=YES - - pwsh: | - $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }}/package-info.json + - pwsh: npm install -g moxygen + displayName: Install Moxygen to generate markdown for docs.microsoft.com - $version = eng/scripts/Get-PkgVersion -ServiceDirectory ${{ parameters.ServiceDirectory }} -PackageName ${{ artifact.Path }} - $outputObject = @{ version = $version.ToString(); name = '${{ artifact.Name }}'; packageName = "${{ artifact.Name }}_$version"; } | ConvertTo-Json + # Generate package spec JSON files for tagging the repo + - ${{ each artifact in parameters.Artifacts }}: + - pwsh: >- + New-Item + -ItemType directory + -Path $(Build.ArtifactStagingDirectory) + -Name packages/${{ artifact.Name }} + displayName: Create package info artifact location - Set-Content -Path $outputPath -Value $outputObject + - pwsh: | + $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }}/package-info.json - # Set AZURE_SDK_VERSION for use in Doxygen generation - Write-Host "##vso[task.setvariable variable=AZURE_SDK_VERSION]$version" - displayName: Create package info JSON file + $version = eng/scripts/Get-PkgVersion -ServiceDirectory ${{ parameters.ServiceDirectory }} -PackageName ${{ artifact.Path }} + $outputObject = @{ version = $version.ToString(); name = '${{ artifact.Name }}'; packageName = "${{ artifact.Name }}_$version"; } | ConvertTo-Json - - task: Powershell@2 - inputs: - targetType: inline - script: | - # If there is no vcpkg directory skip staging the vcpkg port artifacts - if (!(Test-Path build/vcpkg/ports/${{ artifact.VcpkgPortName }})) { - Write-Host "No vcpkg port directory at build/vcpkg/ports/${{ artifact.VcpkgPortName }}" - exit + Set-Content -Path $outputPath -Value $outputObject + + # Set AZURE_SDK_VERSION for use in Doxygen generation + Write-Host "##vso[task.setvariable variable=AZURE_SDK_VERSION]$version" + displayName: Create package info JSON file + + - task: Powershell@2 + inputs: + targetType: inline + script: | + # If there is no vcpkg directory skip staging the vcpkg port artifacts + if (!(Test-Path build/vcpkg/ports/${{ artifact.VcpkgPortName }})) { + Write-Host "No vcpkg port directory at build/vcpkg/ports/${{ artifact.VcpkgPortName }}" + exit + } + + New-Item ` + -ItemType Directory ` + $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg -Force + + Copy-Item -Recurse ` + build/vcpkg/ports/${{ artifact.VcpkgPortName }}/. ` + $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg/port + pwsh: true + displayName: Copy vcpkg port files from build + + - pwsh: | + $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }} + $changelogPath = "sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md" + + if (Test-Path $changeLogPath) { + Copy-Item sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md $outputPath + } else { + Write-Warning "$changeLogPath does not exist" } + displayName: Copy CHANGELOG.md to package artifact - New-Item ` - -ItemType Directory ` - $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg -Force + - script: cmake --build . --target ${{ artifact.Name }}-docs + workingDirectory: build + displayName: Generate docs (${{ artifact.Name }}-docs) - Copy-Item -Recurse ` - build/vcpkg/ports/${{ artifact.VcpkgPortName }}/. ` - $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/vcpkg/port - pwsh: true - displayName: Copy vcpkg port files from build + - task: PowerShell@2 + inputs: + targetType: filePath + filePath: eng/scripts/New-DocsMsArtifact.ps1 + arguments: >- + -ServiceDirectory ${{ parameters.ServiceDirectory }} + -PackageName ${{ artifact.Name }} + -TargetFolder $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/docs/docs.ms + ignoreLASTEXITCODE: true + pwsh: true + displayName: Generate artifacts for docs.ms - - pwsh: | - $outputPath = Join-Path -Path $(Build.ArtifactStagingDirectory) packages/${{ artifact.Name }} - $changelogPath = "sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md" + - pwsh: | + New-Item -ItemType directory -Path $(Build.ArtifactStagingDirectory) -Name docs/${{ artifact.Name }} + displayName: Create doc artifact location - if (Test-Path $changeLogPath) { - Copy-Item sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/CHANGELOG.md $outputPath - } else { - Write-Warning "$changeLogPath does not exist" - } - displayName: Copy CHANGELOG.md to package artifact + - task: CopyFiles@2 + inputs: + sourceFolder: $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }} + contents: package-info.json + targetFolder: build/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/docs/html + displayName: Copy package-info.json to documentation path - - script: cmake --build . --target ${{ artifact.Name }}-docs - workingDirectory: build - displayName: Generate docs (${{ artifact.Name }}-docs) + - task: CopyFiles@2 + inputs: + sourceFolder: build/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/docs/html + targetFolder: $(Build.ArtifactStagingDirectory)/docs/${{ artifact.Name }} + displayName: Copy documentation to artifact staging directory - - task: PowerShell@2 - inputs: - targetType: filePath - filePath: eng/scripts/New-DocsMsArtifact.ps1 - arguments: >- - -ServiceDirectory ${{ parameters.ServiceDirectory }} - -PackageName ${{ artifact.Name }} - -TargetFolder $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }}/docs/docs.ms - ignoreLASTEXITCODE: true - pwsh: true - displayName: Generate artifacts for docs.ms + - task: PublishPipelineArtifact@1 + inputs: + artifactName: packages + path: $(Build.ArtifactStagingDirectory)/packages - - pwsh: | - New-Item -ItemType directory -Path $(Build.ArtifactStagingDirectory) -Name docs/${{ artifact.Name }} - displayName: Create doc artifact location + # After all docs artifacts are generated publish docs artifacts + - task: PublishPipelineArtifact@1 + inputs: + artifactName: docs + path: $(Build.ArtifactStagingDirectory)/docs - - task: CopyFiles@2 - inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/packages/${{ artifact.Name }} - contents: package-info.json - targetFolder: build/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/docs/html - displayName: Copy package-info.json to documentation path - - - task: CopyFiles@2 - inputs: - sourceFolder: build/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.Path }}/docs/html - targetFolder: $(Build.ArtifactStagingDirectory)/docs/${{ artifact.Name }} - displayName: Copy documentation to artifact staging directory - - - task: PublishPipelineArtifact@1 - inputs: - artifactName: packages - path: $(Build.ArtifactStagingDirectory)/packages - - # After all docs artifacts are generated publish docs artifacts - - task: PublishPipelineArtifact@1 - inputs: - artifactName: docs - path: $(Build.ArtifactStagingDirectory)/docs - - - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml + - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml diff --git a/eng/pipelines/templates/jobs/archetype-sdk-tests.yml b/eng/pipelines/templates/jobs/archetype-sdk-tests.yml index a9a97a243..d236f3cc3 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-tests.yml @@ -21,6 +21,11 @@ parameters: type: number default: 60 +# This job uses the legacy matrix format (matrix property of the job, one +# build/test scenario per job). A new format (multiple build/test scenarios per +# job) may be possible here: +# https://github.com/Azure/azure-sdk-for-cpp/issues/2944 + jobs: - job: Validate condition: and(succeededOrFailed(), ne(variables['Skip.LiveTest'], 'true')) @@ -68,7 +73,6 @@ jobs: Win_x86_with_unit_test_winHttp: Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 - VcpkgInstall: 'libxml2' VCPKG_DEFAULT_TRIPLET: 'x86-windows-static' CMAKE_GENERATOR: 'Visual Studio 16 2019' CMAKE_GENERATOR_PLATFORM: Win32 @@ -87,7 +91,6 @@ jobs: Win_x64_with_unit_test_winHttp: Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 - VcpkgInstall: 'libxml2' VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' CMAKE_GENERATOR: 'Visual Studio 16 2019' CMAKE_GENERATOR_PLATFORM: x64 @@ -98,7 +101,7 @@ jobs: Win_x86_with_unit_test_libcurl: Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' + VcpkgInstall: 'curl[winssl]' VCPKG_DEFAULT_TRIPLET: 'x86-windows-static' CMAKE_GENERATOR: 'Visual Studio 16 2019' CMAKE_GENERATOR_PLATFORM: Win32 @@ -108,7 +111,7 @@ jobs: Win_x64_with_unit_test_libcurl: Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 - VcpkgInstall: 'curl[winssl] libxml2' + VcpkgInstall: 'curl[winssl]' VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' CMAKE_GENERATOR: 'Visual Studio 16 2019' CMAKE_GENERATOR_PLATFORM: x64 diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml new file mode 100644 index 000000000..fc3c0fcc4 --- /dev/null +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -0,0 +1,138 @@ +parameters: + - name: Artifacts + type: object + default: [] + - name: ServiceDirectory + type: string + default: not-specified + - name: TestPipeline + type: boolean + default: false + - name: CtestRegex + type: string + default: .* + - name: CoverageReportPath + type: string + default: 'sdk/*/*/*cov_xml.xml' + - name: CoverageEnabled + type: boolean + default: true + - name: Matrix + type: string + + # Supplied by archetype-sdk-tests-generate.yml dynamic matrix generator + - name: CloudConfig + type: object + default: {} + # Supplied by archetype-sdk-tests-generate.yml dynamic matrix generator. + # Must be wired up to ensure population of parameters.Matrix + - name: DependsOn + type: string + default: '' + # Supplied by archetype-sdk-tests-generate.yml dynamic matrix generator + - name: UsePlatformContainer + type: boolean + default: false + +jobs: + - job: + displayName: "Validate" + dependsOn: ${{ parameters.DependsOn }} + condition: and(succeededOrFailed(), ne(variables['Skip.Test'], 'true'), ne(${{ parameters.Matrix }}, '{}')) + strategy: + matrix: $[ ${{ parameters.Matrix }} ] + maxParallel: 12 + pool: + vmImage: $(OSVmImage) + name: $(Pool) + variables: + - name: CMOCKA_XML_FILE + value: "%g-test-results.xml" + - name: CMOCKA_MESSAGE_OUTPUT + value: "xml" + - name: BuildArgs + value: "" + - name: CmakeEnvArg + value: "" + - name: CmakeArgs + value: "" + - name: AZURE_TEST_MODE + value: "PLAYBACK" + steps: + - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml + parameters: + AgentImage: $(OsVmImage) + + - pwsh: sudo apt update && sudo apt install -y $(AptDependencies) + condition: and(succeeded(), ne(variables['AptDependencies'], '')) + displayName: Install dependencies from apt + + - template: /eng/pipelines/templates/steps/vcpkg.yml + parameters: + DependenciesVariableName: VcpkgInstall + + # Validate all the files are formatted correctly according to the + # .clang-format file. This step runs on linux only only and assumes that + # clang-format-11 is installed. + - bash: | + # Run clang-format recursively on each source and header file within the repo sdk folder. + echo "Check clang-formatting" + clang-format --version + find ./sdk \( -iname '*.hpp' -o -iname '*.cpp' \) ! -iname 'json.hpp' -exec clang-format -i {} \; + + if [[ `git status | grep modified | awk '{print $2}'` ]]; then + echo Some files were not formatted correctly according to the .clang-format file. + echo Please run clang-format version 10 or greater to fix the issue by using this bash command at the root of the repo: + echo "find ./sdk \( -iname '*.hpp' -o -iname '*.cpp' \) ! -iname 'json.hpp' -exec clang-format -i {} \;" + echo "" + echo "List of files not formatted correctly:" + git status | grep modified | awk '{print $2}' + exit 1 + fi + + echo Success, all files are formatted correctly according to the .clang-format file. + exit 0 + + displayName: Validate Clang Format + condition: and(succeededOrFailed(), eq(variables['CHECK_CLANG_FORMAT'], 1)) + + - ${{ each artifact in parameters.Artifacts }}: + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: ${{ artifact.Name }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestPipeline: ${{ parameters.TestPipeline }} + + - template: /eng/pipelines/templates/steps/cmake-build.yml + parameters: + GenerateArgs: "$(CmakeArgs)" + BuildArgs: "$(BuildArgs)" + Env: "$(CmakeEnvArg)" + + - script: ctest -C Debug -V --tests-regex ${{ parameters.CtestRegex }} + workingDirectory: build + + - ${{ if eq(parameters.CoverageEnabled, true) }}: + - pwsh: | + $toolsDirectory = "$(Agent.TempDirectory)/coveragetools" + dotnet tool install -g dotnet-reportgenerator-globaltool + dotnet tool install dotnet-reportgenerator-globaltool --tool-path $toolsDirectory + Write-Host "##vso[task.setvariable variable=ToolsDirectory]$toolsDirectory" + displayName: Install coverage tools + condition: and(succeeded(), eq(variables['CODE_COVERAGE'], 'enabled')) + + # Make coverage targets (specified in coverage_targets.txt) and assemble + # coverage report + - bash: | + make VERBOSE=1 `cat ${{ parameters.ServiceDirectory }}-targets-coverage.txt` + $(ToolsDirectory)/reportgenerator "-reports:${{ parameters.CoverageReportPath }}" "-targetdir:." "-reporttypes:Cobertura" + workingDirectory: build + displayName: Generate Code Coverage Data + condition: and(succeededOrFailed(), eq(variables['CODE_COVERAGE'], 'enabled')) + + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(Build.SourcesDirectory)/**/Cobertura.xml' + displayName: Publish Code Coverage to DevOps + condition: and(succeededOrFailed(), eq(variables['CODE_COVERAGE'], 'enabled')) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 14aa8ce92..0c9eb5831 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -5,9 +5,9 @@ parameters: - name: CtestRegex type: string default: .* -- name: Coverage - type: string - default: 'enabled' +- name: CoverageEnabled + type: boolean + default: true - name: LiveTestCtestRegex type: string default: '' @@ -35,7 +35,7 @@ stages: ServiceDirectory: ${{ parameters.ServiceDirectory }} Artifacts: ${{ parameters.Artifacts }} CtestRegex: ${{ parameters.CtestRegex }} - Coverage: ${{ parameters.Coverage }} + CoverageEnabled: ${{ parameters.CoverageEnabled }} CoverageReportPath: ${{ parameters.CoverageReportPath }} ${{ if eq(parameters.ServiceDirectory, 'template') }}: TestPipeline: true diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json new file mode 100644 index 000000000..f58ee8dd5 --- /dev/null +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -0,0 +1,154 @@ +{ + "displayNames": { + "_": "" + }, + "include": [ + { + "OSVmImage": ["macOS-10.14", "macOS-10.15", "macOS-11"], + "StaticConfigs": { + "_": { + "Pool": "Azure Pipelines", + "VcpkgInstall": "curl[ssl] libxml2 openssl", + "BuildArgs": "-j 10", + "VCPKG_DEFAULT_TRIPLET": "x64-osx", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON" + } + }, + "BuildConfig": { + "debug": { + "CMAKE_BUILD_TYPE": "Debug" + }, + "release": { + "CMAKE_BUILD_TYPE": "Release" + } + } + }, + { + "StaticConfigs": { + "Windows2019": { + "OSVmImage": "MMS2019", + "Pool": "azsdk-pool-mms-win-2019-general", + "BuildArgs": "--parallel 8", + "CMAKE_GENERATOR": "Visual Studio 16 2019" + } + }, + "TargetPlatform": { + "Win32Api_curl": { + "VcpkgInstall": "curl[winssl]", + "CmakeArgs": " -DBUILD_TRANSPORT_CURL=ON" + }, + "Win32Api_release_curl": { + "VcpkgInstall": "curl[winssl]", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON", + "CMAKE_BUILD_TYPE": "Release" + }, + "Win32Api_debug_tests": { + "VcpkgInstall": "curl[winssl]", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_TRANSPORT_WINHTTP=ON" + } + }, + "TargetArchitecture": { + "x86": { + "CMAKE_GENERATOR_PLATFORM": "Win32", + "VCPKG_DEFAULT_TRIPLET": "x86-windows-static" + }, + "x64": { + "CMAKE_GENERATOR_PLATFORM": "x64", + "VCPKG_DEFAULT_TRIPLET": "x64-windows-static" + } + } + }, + { + "StaticConfigs": { + "Windows2019": { + "OSVmImage": "MMS2019", + "Pool": "azsdk-pool-mms-win-2019-general", + "BuildArgs": "--parallel 8", + "CMAKE_GENERATOR": "Visual Studio 16 2019", + "CmakeArgs": " -DBUILD_TRANSPORT_WINHTTP=ON -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON" + } + }, + "TargetPlatform": { + "UWP_debug": { + "CMAKE_SYSTEM_NAME": "WindowsStore", + "CMAKE_SYSTEM_VERSION": "10.0" + }, + "UWP_release": { + "CMAKE_SYSTEM_NAME": "WindowsStore", + "CMAKE_SYSTEM_VERSION": "10.0", + "CMAKE_BUILD_TYPE": "Release" + } + }, + "TargetArchitecture": { + "x86": { + "CMAKE_GENERATOR_PLATFORM": "Win32" + }, + "x64": { + "CMAKE_GENERATOR_PLATFORM": "x64", + "VCPKG_DEFAULT_TRIPLET": "x64-uwp" + } + } + }, + { + "StaticConfigs": { + "Ubuntu18": { + "OSVmImage": "MMSUbuntu18.04", + "Pool": "azsdk-pool-mms-ubuntu-1804-general", + "VcpkgInstall": "curl[ssl] libxml2 openssl", + "VCPKG_DEFAULT_TRIPLET": "x64-linux", + "BuildArgs": "-j 10" + } + }, + "BuildSettings": { + "gpp-5": { + "AptDependencies": "g++-5", + "CmakeEnvArg": "CC=/usr/bin/gcc-5 CXX=/usr/bin/g++-5 " + }, + "gpp-8": { + "AptDependencies": "g++-8", + "CC": "/usr/bin/gcc-8", + "CXX": "/usr/bin/g++-8" + }, + "gpp-9": { + "AptDependencies": "g++-9", + "CC": "/usr/bin/gcc-9", + "CXX": "/usr/bin/g++-9" + }, + "included_coverage": { + "AptDependencies": "gcovr lcov", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CODE_COVERAGE=ON -DCMAKE_VERBOSE_MAKEFILE=ON", + "CODE_COVERAGE_COLLECT_ONLY": "1", + "CODE_COVERAGE": "enabled" + }, + "included": { }, + "included_release": { + "CMAKE_BUILD_TYPE": "Release", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON" + } + } + }, + { + "StaticConfigs": { + "Ubuntu20": { + "OSVmImage": "MMSUbuntu20.04", + "Pool": "azsdk-pool-mms-ubuntu-2004-general", + "VcpkgInstall": "curl[ssl] libxml2 openssl", + "VCPKG_DEFAULT_TRIPLET": "x64-linux", + "BuildArgs": "-j 10", + "CC": "/usr/bin/clang-11", + "CXX": "/usr/bin/clang++-11", + "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON" + } + }, + "BuildSettings": { + "clang-11": { + "CHECK_CLANG_FORMAT": "1" + }, + "included_release": { + "CMAKE_BUILD_TYPE": "Release" + + } + } + } + ] +} diff --git a/eng/pipelines/templates/steps/vcpkg.yml b/eng/pipelines/templates/steps/vcpkg.yml index 9eee8179f..94097e562 100644 --- a/eng/pipelines/templates/steps/vcpkg.yml +++ b/eng/pipelines/templates/steps/vcpkg.yml @@ -25,7 +25,7 @@ steps: ) # This task times out after 10 minutes. An analysis of a subset of Vcpkg Cache - # runs showed that after 10 minutes the cache step is very unlikely to + # runs showed that after 10 minutes the cache step is very unlikely to # succeed. - task: Cache@2 inputs: @@ -33,7 +33,6 @@ steps: $(Agent.JobName) | "$(VcpkgCommit)" | $(Agent.Os) - | $(${{ parameters.DependenciesVariableName }}) path: $(VCPKG_INSTALLATION_ROOT) cacheHitVar: VcpkgRestoredFromCache displayName: Vcpkg Cache diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index b931d800f..5835f5873 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -37,7 +37,7 @@ stages: ServiceDirectory: storage # Not running tests on CI but only on LiveTest mode CtestRegex: no-run - Coverage: disabled + CoverageEnabled: false LiveTestCtestRegex: azure-storage SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview) Artifacts: