diff --git a/CMakeLists.txt b/CMakeLists.txt index d761950bf..dddbc9699 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,20 +9,13 @@ option(WARNINGS_AS_ERRORS "Treat compiler warnings as errors" ON) option(BUILD_TRANSPORT_CURL "Build an HTTP transport implementation with CURL" OFF) option(BUILD_TRANSPORT_WINHTTP "Build an HTTP transport implementation with WIN HTTP" OFF) option(BUILD_TESTING "Build test cases" OFF) +option(BUILD_CODE_COVERAGE "Build gcov targets for HTML and XML reports. Requires debug build and BUILD_TESTING" OFF) option(BUILD_DOCUMENTATION "Create HTML based API documentation (requires Doxygen)" OFF) option(RUN_LONG_UNIT_TESTS "Tests that takes more than 5 minutes to complete. No effect if BUILD_TESTING is OFF" OFF) option(BUILD_STORAGE_SAMPLES "Build sample application for Azure Storage clients" OFF) include(DefineTransportAdapter) -if(BUILD_TESTING) - # define a symbol that enables some test hooks in code - add_compile_definitions(TESTING_BUILD) - if(RUN_LONG_UNIT_TESTS) - add_compile_definitions(RUN_LONG_UNIT_TESTS) - endif() -endif() - # VCPKG Integration if(DEFINED ENV{VCPKG_ROOT} AND NOT DEFINED CMAKE_TOOLCHAIN_FILE) set(CMAKE_TOOLCHAIN_FILE "$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" @@ -44,6 +37,12 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED True) if(BUILD_TESTING) + # define a symbol that enables some test hooks in code + add_compile_definitions(TESTING_BUILD) + if(RUN_LONG_UNIT_TESTS) + add_compile_definitions(RUN_LONG_UNIT_TESTS) + endif() + # tests include(AddGoogleTest) enable_testing () diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c9101e457..c97e7d9fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,6 +111,11 @@ The following CMake options are available for adding/removing project features. OFF +BUILD_CODE_COVERAGE +Build HTML and XML targets for each package which can be call to produce XML or HTML reports. The generated CMake targets are named `package-name_cov_xml` and `package-name_cov_html` (for example, for Azure Core, it would be `azure-core-cov_xml`).

Ths option requires compiling on `debug` mode, building tests (BUILD_TESTING) and a GNU compiler like gcc. +OFF + + BUILD_STORAGE_SAMPLES Build Azure Storage clients sample application. OFF @@ -126,7 +131,7 @@ The following CMake options are available for adding/removing project features. ON -BUILD_CURL_TRANSPORT +BUILD_TRANSPORT_CURL Build the curl http transport adapter. When building on Posix systems, if no other transport adapter is built, this option will be automatically turned ON OFF @@ -153,6 +158,24 @@ ctest -N # Use -R to use a regular exp for what to run ctest -R Http # runs only Http tests ``` +#### Generating Code Coverage reports +`gcov` and `gcovr` must be installed on your system. +Also, make sure to generate the project with Debug mode. Then, option `-DBUILD_TESTING` must be `ON` and to use a GNU compiler (like gcc). + +```sh +# install gcov and gcovr if missing +sudo apt-get install gcov gcovr # example for Linux + +cmake -DBUILD_TESTING=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CODE_COVERAGE=ON .. + +# After this, generate reports by calling a package target +make package-name_cov_xml # for example `azure-core_cov_xml` +make package-name_cov_html # for example `azure-core_cov_html` +``` + +Running the above commands will create the test executable and run it. While it runs, gcov will capture coverage and produce coverage data. And when test finished, gcovr is used to parse the coverage data to produce and XML or HTML. The output files will be inside the package build directory. For example, for Azure core, it will be `../build/path/sdk/core/azure-core/`. + +If the coverage data has been previously generated (for example, if you manually run the unit tests), you can define `AZURE_CI_TEST` environment variable (set it to any value) and then the report will be generated without running the tests again. This is how the coverage reports are generated on CI, where the tests runs prior to code coverage step. ### Visual Studio 2019 You can also build the project by simply opening the repo directory in Visual Studio. Visual Studio will detect the `CMake` file and will configure itself to generate, build and run tests. diff --git a/cmake-modules/CodeCoverage.cmake b/cmake-modules/CodeCoverage.cmake new file mode 100644 index 000000000..791f1d130 --- /dev/null +++ b/cmake-modules/CodeCoverage.cmake @@ -0,0 +1,118 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Defines utility functions to create code coverage targets with gcov. +# gcov html and xml report. +# + +## SET UP only when option is set. +if(BUILD_CODE_COVERAGE) + if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(FATAL_ERROR "Code coverage requires GNU compiler (gcc) and debug build type. ${CMAKE_CXX_COMPILER_ID}") + endif() + + ## Check if gcov and gcovr are available (gcovr generates html and xml from gcov files) + find_program(GCOV gcov) + find_program(GCOVR_PATH gcovr PATHS ${CMAKE_SOURCE_DIR}/scripts/test) + if(NOT GCOV) + message(FATAL_ERROR "gcov is required for code coverage reports.") + endif() + link_libraries(gcov) +endif() + +# Set the compiler flags for the CMake project from where this is called (PARENT_SCOPE) +function(append_code_coverage_for_current_project) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fprofile-arcs -ftest-coverage" PARENT_SCOPE) +endfunction() + + +# gcovr - html +function(add_gcovr_html) + + set(options NONE) + set(oneValueArgs TARGET_NAME EXECUTABLE_NAME) + set(multiValueArgs NA) + cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(BASEDIR ${PROJECT_SOURCE_DIR}) + + add_custom_target(${args_TARGET_NAME} + # Run tests + ${args_EXECUTABLE_NAME} + + # Create folder + COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${args_TARGET_NAME} + + # Running gcovr + COMMAND ${GCOVR_PATH} --html --html-details + -r ${BASEDIR} + --object-directory=${PROJECT_BINARY_DIR} + -o ${args_TARGET_NAME}/index.html + + BYPRODUCTS ${PROJECT_BINARY_DIR}/${args_TARGET_NAME} # report directory + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + DEPENDS + VERBATIM + COMMENT "Running gcovr to produce HTML code coverage report." + ) + + # Show info where to find the report + add_custom_command(TARGET ${args_TARGET_NAME} POST_BUILD + COMMAND ; + COMMENT "Open ./${args_TARGET_NAME}/index.html in your browser to view the coverage report." + ) + +endfunction() + +# gcovr - xml +function(add_gcovr_xml) + + set(options NONE) + set(oneValueArgs TARGET_NAME EXECUTABLE_NAME) + set(multiValueArgs NA) + cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(BASEDIR ${PROJECT_SOURCE_DIR}) + + if (NOT DEFINED ENV{AZURE_CI_TEST}) + set(RUN_EXE ${args_EXECUTABLE_NAME}) + endif() + + add_custom_target(${args_TARGET_NAME} + # Running on CI won't require to run tests exe since it was run on previous step + ${RUN_EXE} + + # Running gcovr + COMMAND ${GCOVR_PATH} --xml + -r ${BASEDIR} + --object-directory=${PROJECT_BINARY_DIR} + -o ${args_TARGET_NAME}.xml + BYPRODUCTS ${args_TARGET_NAME}.xml + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + DEPENDS + VERBATIM + COMMENT "Running gcovr to produce Cobertura code coverage report." + ) + + # Show info where to find the report + add_custom_command(TARGET ${args_TARGET_NAME} POST_BUILD + COMMAND ; + COMMENT "Cobertura code coverage report saved in ${args_TARGET_NAME}.xml." + ) + +endfunction() + +# codeCoverage macro to be used from CMake lib definition +macro(create_code_coverage service target_prefix exe_name) + if(BUILD_CODE_COVERAGE) + APPEND_CODE_COVERAGE_FOR_CURRENT_PROJECT() + + # HTML and XML - Coverage using gcovr + add_gcovr_html(TARGET_NAME ${target_prefix}_cov_html EXECUTABLE_NAME ${exe_name}) + # xml is used on CI + add_gcovr_xml(TARGET_NAME ${target_prefix}_cov_xml EXECUTABLE_NAME ${exe_name}) + + # add xml target to `coverage_targets.txt` which is used by CI to generate coverage reports + file(APPEND ${CMAKE_BINARY_DIR}/${service}-targets-coverage.txt " ${target_prefix}_cov_xml") + endif() +endmacro() diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index ec02785dc..b9dad6c7a 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -1,6 +1,7 @@ parameters: Artifacts: [] ServiceDirectory: not-specified + Coverage: '' CtestRegex: .* BuildReleaseArtifacts: true @@ -15,16 +16,19 @@ jobs: VCPKG_DEFAULT_TRIPLET: 'x64-linux' CC: '/usr/bin/gcc-8' CXX: '/usr/bin/g++-8' + BuildArgs: '-j 10' Linux_x64_gcc9: OSVmImage: 'ubuntu-18.04' VcpkgInstall: 'curl[ssl] libxml2 openssl' VCPKG_DEFAULT_TRIPLET: 'x64-linux' CC: '/usr/bin/gcc-9' CXX: '/usr/bin/g++-9' + BuildArgs: '-j 10' Linux_x64: OSVmImage: 'ubuntu-18.04' VcpkgInstall: 'curl[ssl] libxml2 openssl' VCPKG_DEFAULT_TRIPLET: 'x64-linux' + BuildArgs: '-j 10' Win_x86: OSVmImage: 'windows-2019' VcpkgInstall: 'curl[winssl] libxml2' @@ -49,7 +53,12 @@ jobs: OSVmImage: 'ubuntu-18.04' VcpkgInstall: 'curl[ssl] libxml2 openssl' VCPKG_DEFAULT_TRIPLET: 'x64-linux' - CmakeArgs: ' -DBUILD_TESTING=ON -DRUN_LONG_UNIT_TESTS=ON' + CmakeArgs: ' -DBUILD_TESTING=ON -DRUN_LONG_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CODE_COVERAGE=ON' + AptDependencies: 'gcovr lcov' + CODE_COVERAGE: '${{ parameters.Coverage }}' + # Make coverage report to avoid running the test exe because CI step will run it + AZURE_CI_TEST: 1 + BuildArgs: '-j 10' Win_x86_with_unit_test: OSVmImage: 'windows-2019' VcpkgInstall: 'curl[winssl] libxml2' @@ -74,23 +83,54 @@ jobs: variables: CMOCKA_XML_FILE: "%g-test-results.xml" CMOCKA_MESSAGE_OUTPUT: "xml" + BuildArgs: "" steps: - checkout: self submodules: recursive + # Install apt dependencies (if appropriate) + - bash: sudo apt install -y $(AptDependencies) + condition: and(succeededOrFailed(), ne(variables['AptDependencies'], '')) + displayName: Install dependencies from apt + - template: /eng/pipelines/templates/steps/vcpkg.yml parameters: DependenciesVariableName: VcpkgInstall + - script: | + dotnet tool install -g dotnet-reportgenerator-globaltool + dotnet tool install dotnet-reportgenerator-globaltool --tool-path tools + displayName: Install coverage tools + # CODE_COVERAGE variable is '' (do-not-generate) in all matrix but linux-gcc + # It is 'enabled' by default on linux-gcc but it can be opt-out by each pipeline (disabled) + condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], '')) + - template: /eng/pipelines/templates/steps/cmake-build.yml parameters: - GenerateArgs: $(CmakeArgs) + GenerateArgs: "$(CmakeArgs)" + BuildArgs: "$(BuildArgs)" - script: ctest -C Debug -V --tests-regex ${{ parameters.CtestRegex }} workingDirectory: build displayName: ctest + # Make coverage targets (specified in coverage_targets.txt) and assemble + # coverage report + - bash: | + make `cat ${{ parameters.ServiceDirectory }}-targets-coverage.txt` + ../tools/reportgenerator "-reports:sdk/*/*/*cov_xml.xml" "-targetdir:." "-reporttypes:Cobertura" + workingDirectory: build + displayName: Generate Code Coverage Data + condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], '')) + + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(Build.SourcesDirectory)/**/Cobertura.xml' + displayName: Publish Code Coverage to DevOps + condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], '')) + # Disable build for cpp - client - ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}: - job: GenerateReleaseArtifacts diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 9e1f75922..39c332705 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -8,6 +8,9 @@ parameters: - name: CtestRegex type: string default: .* +- name: Coverage + type: string + default: 'enabled' stages: @@ -18,6 +21,7 @@ stages: ServiceDirectory: ${{ parameters.ServiceDirectory }} Artifacts: ${{ parameters.Artifacts }} CtestRegex: ${{ parameters.CtestRegex }} + Coverage: ${{ parameters.Coverage }} - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: diff --git a/eng/pipelines/templates/steps/cmake-build.yml b/eng/pipelines/templates/steps/cmake-build.yml index 35098bf4b..19d52cddd 100644 --- a/eng/pipelines/templates/steps/cmake-build.yml +++ b/eng/pipelines/templates/steps/cmake-build.yml @@ -17,6 +17,6 @@ steps: displayName: cmake generate - ${{ if eq(parameters.Build, true) }}: - - script: cmake ${{ parameters.BuildArgs }} --build . + - script: cmake --build . ${{ parameters.BuildArgs }} workingDirectory: build - displayName: cmake build \ No newline at end of file + displayName: cmake build diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index b86c8f474..372d857f4 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -14,6 +14,7 @@ ### Other changes and Improvements - Add high-level and simplified core.hpp file for simpler include experience for customers. +- Add code coverage using gcov with gcc. - Update SDK-defined exception types to be classes instead of structs. ### Bug Fixes diff --git a/sdk/core/azure-core/CMakeLists.txt b/sdk/core/azure-core/CMakeLists.txt index 24957a008..3373bb12d 100644 --- a/sdk/core/azure-core/CMakeLists.txt +++ b/sdk/core/azure-core/CMakeLists.txt @@ -21,6 +21,9 @@ if(BUILD_TRANSPORT_WINHTTP) SET(WIN_TRANSPORT_ADAPTER_SRC src/http/winhttp/win_http_transport.cpp) endif() +# CodeCoverage will validate requirements +include(CodeCoverage) + add_library ( ${TARGET_NAME} src/context.cpp @@ -48,6 +51,9 @@ target_include_directories (${TARGET_NAME} PUBLIC $) +# coverage. Has no effect if BUILD_CODE_COVERAGE is OFF +create_code_coverage(template azure-template azure-template-test) + # make sure that users can consume the project as a library. add_library (Azure::Template ALIAS azure-template)