Add Code coverage (#830)

* Add code coverage scripts
This commit is contained in:
Victor Vazquez 2020-10-27 15:35:15 -07:00 committed by GitHub
parent 7f46381cdd
commit c764052e7c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 242 additions and 13 deletions

View File

@ -9,20 +9,13 @@ option(WARNINGS_AS_ERRORS "Treat compiler warnings as errors" ON)
option(BUILD_TRANSPORT_CURL "Build an HTTP transport implementation with CURL" OFF)
option(BUILD_TRANSPORT_WINHTTP "Build an HTTP transport implementation with WIN HTTP" OFF)
option(BUILD_TESTING "Build test cases" OFF)
option(BUILD_CODE_COVERAGE "Build gcov targets for HTML and XML reports. Requires debug build and BUILD_TESTING" OFF)
option(BUILD_DOCUMENTATION "Create HTML based API documentation (requires Doxygen)" OFF)
option(RUN_LONG_UNIT_TESTS "Tests that takes more than 5 minutes to complete. No effect if BUILD_TESTING is OFF" OFF)
option(BUILD_STORAGE_SAMPLES "Build sample application for Azure Storage clients" OFF)
include(DefineTransportAdapter)
if(BUILD_TESTING)
# define a symbol that enables some test hooks in code
add_compile_definitions(TESTING_BUILD)
if(RUN_LONG_UNIT_TESTS)
add_compile_definitions(RUN_LONG_UNIT_TESTS)
endif()
endif()
# VCPKG Integration
if(DEFINED ENV{VCPKG_ROOT} AND NOT DEFINED CMAKE_TOOLCHAIN_FILE)
set(CMAKE_TOOLCHAIN_FILE "$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
@ -44,6 +37,12 @@ set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED True)
if(BUILD_TESTING)
# define a symbol that enables some test hooks in code
add_compile_definitions(TESTING_BUILD)
if(RUN_LONG_UNIT_TESTS)
add_compile_definitions(RUN_LONG_UNIT_TESTS)
endif()
# tests
include(AddGoogleTest)
enable_testing ()

View File

@ -111,6 +111,11 @@ The following CMake options are available for adding/removing project features.
<td>OFF</td>
</tr>
<tr>
<td>BUILD_CODE_COVERAGE</td>
<td>Build HTML and XML targets for each package which can be call to produce XML or HTML reports. The generated CMake targets are named `package-name_cov_xml` and `package-name_cov_html` (for example, for Azure Core, it would be `azure-core-cov_xml`).<br> <br>Ths option requires compiling on `debug` mode, building tests (BUILD_TESTING) and a GNU compiler like gcc. </td>
<td>OFF</td>
</tr>
<tr>
<td>BUILD_STORAGE_SAMPLES</td>
<td>Build Azure Storage clients sample application.</td>
<td>OFF</td>
@ -126,7 +131,7 @@ The following CMake options are available for adding/removing project features.
<td>ON</td>
</tr>
<tr>
<td>BUILD_CURL_TRANSPORT</td>
<td>BUILD_TRANSPORT_CURL</td>
<td>Build the curl http transport adapter. When building on Posix systems, if no other transport adapter is built, this option will be automatically turned ON</td>
<td>OFF</td>
</tr>
@ -153,6 +158,24 @@ ctest -N
# Use -R to use a regular exp for what to run
ctest -R Http # runs only Http tests
```
#### Generating Code Coverage reports
`gcov` and `gcovr` must be installed on your system.
Also, make sure to generate the project with Debug mode. Then, option `-DBUILD_TESTING` must be `ON` and to use a GNU compiler (like gcc).
```sh
# install gcov and gcovr if missing
sudo apt-get install gcov gcovr # example for Linux
cmake -DBUILD_TESTING=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CODE_COVERAGE=ON ..
# After this, generate reports by calling a package target
make package-name_cov_xml # for example `azure-core_cov_xml`
make package-name_cov_html # for example `azure-core_cov_html`
```
Running the above commands will create the test executable and run it. While it runs, gcov will capture coverage and produce coverage data. And when test finished, gcovr is used to parse the coverage data to produce and XML or HTML. The output files will be inside the package build directory. For example, for Azure core, it will be `../build/path/sdk/core/azure-core/`.
If the coverage data has been previously generated (for example, if you manually run the unit tests), you can define `AZURE_CI_TEST` environment variable (set it to any value) and then the report will be generated without running the tests again. This is how the coverage reports are generated on CI, where the tests runs prior to code coverage step.
### Visual Studio 2019
You can also build the project by simply opening the repo directory in Visual Studio. Visual Studio will detect the `CMake` file and will configure itself to generate, build and run tests.

View File

@ -0,0 +1,118 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# SPDX-License-Identifier: MIT
#
# Defines utility functions to create code coverage targets with gcov.
# gcov html and xml report.
#
## SET UP only when option is set.
if(BUILD_CODE_COVERAGE)
if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
message(FATAL_ERROR "Code coverage requires GNU compiler (gcc) and debug build type. ${CMAKE_CXX_COMPILER_ID}")
endif()
## Check if gcov and gcovr are available (gcovr generates html and xml from gcov files)
find_program(GCOV gcov)
find_program(GCOVR_PATH gcovr PATHS ${CMAKE_SOURCE_DIR}/scripts/test)
if(NOT GCOV)
message(FATAL_ERROR "gcov is required for code coverage reports.")
endif()
link_libraries(gcov)
endif()
# Set the compiler flags for the CMake project from where this is called (PARENT_SCOPE)
function(append_code_coverage_for_current_project)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fprofile-arcs -ftest-coverage" PARENT_SCOPE)
endfunction()
# gcovr - html
function(add_gcovr_html)
set(options NONE)
set(oneValueArgs TARGET_NAME EXECUTABLE_NAME)
set(multiValueArgs NA)
cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
set(BASEDIR ${PROJECT_SOURCE_DIR})
add_custom_target(${args_TARGET_NAME}
# Run tests
${args_EXECUTABLE_NAME}
# Create folder
COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${args_TARGET_NAME}
# Running gcovr
COMMAND ${GCOVR_PATH} --html --html-details
-r ${BASEDIR}
--object-directory=${PROJECT_BINARY_DIR}
-o ${args_TARGET_NAME}/index.html
BYPRODUCTS ${PROJECT_BINARY_DIR}/${args_TARGET_NAME} # report directory
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
DEPENDS
VERBATIM
COMMENT "Running gcovr to produce HTML code coverage report."
)
# Show info where to find the report
add_custom_command(TARGET ${args_TARGET_NAME} POST_BUILD
COMMAND ;
COMMENT "Open ./${args_TARGET_NAME}/index.html in your browser to view the coverage report."
)
endfunction()
# gcovr - xml
function(add_gcovr_xml)
set(options NONE)
set(oneValueArgs TARGET_NAME EXECUTABLE_NAME)
set(multiValueArgs NA)
cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
set(BASEDIR ${PROJECT_SOURCE_DIR})
if (NOT DEFINED ENV{AZURE_CI_TEST})
set(RUN_EXE ${args_EXECUTABLE_NAME})
endif()
add_custom_target(${args_TARGET_NAME}
# Running on CI won't require to run tests exe since it was run on previous step
${RUN_EXE}
# Running gcovr
COMMAND ${GCOVR_PATH} --xml
-r ${BASEDIR}
--object-directory=${PROJECT_BINARY_DIR}
-o ${args_TARGET_NAME}.xml
BYPRODUCTS ${args_TARGET_NAME}.xml
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
DEPENDS
VERBATIM
COMMENT "Running gcovr to produce Cobertura code coverage report."
)
# Show info where to find the report
add_custom_command(TARGET ${args_TARGET_NAME} POST_BUILD
COMMAND ;
COMMENT "Cobertura code coverage report saved in ${args_TARGET_NAME}.xml."
)
endfunction()
# codeCoverage macro to be used from CMake lib definition
macro(create_code_coverage service target_prefix exe_name)
if(BUILD_CODE_COVERAGE)
APPEND_CODE_COVERAGE_FOR_CURRENT_PROJECT()
# HTML and XML - Coverage using gcovr
add_gcovr_html(TARGET_NAME ${target_prefix}_cov_html EXECUTABLE_NAME ${exe_name})
# xml is used on CI
add_gcovr_xml(TARGET_NAME ${target_prefix}_cov_xml EXECUTABLE_NAME ${exe_name})
# add xml target to `coverage_targets.txt` which is used by CI to generate coverage reports
file(APPEND ${CMAKE_BINARY_DIR}/${service}-targets-coverage.txt " ${target_prefix}_cov_xml")
endif()
endmacro()

View File

@ -1,6 +1,7 @@
parameters:
Artifacts: []
ServiceDirectory: not-specified
Coverage: ''
CtestRegex: .*
BuildReleaseArtifacts: true
@ -15,16 +16,19 @@ jobs:
VCPKG_DEFAULT_TRIPLET: 'x64-linux'
CC: '/usr/bin/gcc-8'
CXX: '/usr/bin/g++-8'
BuildArgs: '-j 10'
Linux_x64_gcc9:
OSVmImage: 'ubuntu-18.04'
VcpkgInstall: 'curl[ssl] libxml2 openssl'
VCPKG_DEFAULT_TRIPLET: 'x64-linux'
CC: '/usr/bin/gcc-9'
CXX: '/usr/bin/g++-9'
BuildArgs: '-j 10'
Linux_x64:
OSVmImage: 'ubuntu-18.04'
VcpkgInstall: 'curl[ssl] libxml2 openssl'
VCPKG_DEFAULT_TRIPLET: 'x64-linux'
BuildArgs: '-j 10'
Win_x86:
OSVmImage: 'windows-2019'
VcpkgInstall: 'curl[winssl] libxml2'
@ -49,7 +53,12 @@ jobs:
OSVmImage: 'ubuntu-18.04'
VcpkgInstall: 'curl[ssl] libxml2 openssl'
VCPKG_DEFAULT_TRIPLET: 'x64-linux'
CmakeArgs: ' -DBUILD_TESTING=ON -DRUN_LONG_UNIT_TESTS=ON'
CmakeArgs: ' -DBUILD_TESTING=ON -DRUN_LONG_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CODE_COVERAGE=ON'
AptDependencies: 'gcovr lcov'
CODE_COVERAGE: '${{ parameters.Coverage }}'
# Make coverage report to avoid running the test exe because CI step will run it
AZURE_CI_TEST: 1
BuildArgs: '-j 10'
Win_x86_with_unit_test:
OSVmImage: 'windows-2019'
VcpkgInstall: 'curl[winssl] libxml2'
@ -74,23 +83,54 @@ jobs:
variables:
CMOCKA_XML_FILE: "%g-test-results.xml"
CMOCKA_MESSAGE_OUTPUT: "xml"
BuildArgs: ""
steps:
- checkout: self
submodules: recursive
# Install apt dependencies (if appropriate)
- bash: sudo apt install -y $(AptDependencies)
condition: and(succeededOrFailed(), ne(variables['AptDependencies'], ''))
displayName: Install dependencies from apt
- template: /eng/pipelines/templates/steps/vcpkg.yml
parameters:
DependenciesVariableName: VcpkgInstall
- script: |
dotnet tool install -g dotnet-reportgenerator-globaltool
dotnet tool install dotnet-reportgenerator-globaltool --tool-path tools
displayName: Install coverage tools
# CODE_COVERAGE variable is '' (do-not-generate) in all matrix but linux-gcc
# It is 'enabled' by default on linux-gcc but it can be opt-out by each pipeline (disabled)
condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], ''))
- template: /eng/pipelines/templates/steps/cmake-build.yml
parameters:
GenerateArgs: $(CmakeArgs)
GenerateArgs: "$(CmakeArgs)"
BuildArgs: "$(BuildArgs)"
- script: ctest -C Debug -V --tests-regex ${{ parameters.CtestRegex }}
workingDirectory: build
displayName: ctest
# Make coverage targets (specified in coverage_targets.txt) and assemble
# coverage report
- bash: |
make `cat ${{ parameters.ServiceDirectory }}-targets-coverage.txt`
../tools/reportgenerator "-reports:sdk/*/*/*cov_xml.xml" "-targetdir:." "-reporttypes:Cobertura"
workingDirectory: build
displayName: Generate Code Coverage Data
condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], ''))
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: '$(Build.SourcesDirectory)/**/Cobertura.xml'
displayName: Publish Code Coverage to DevOps
condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], ''))
# Disable build for cpp - client
- ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}:
- job: GenerateReleaseArtifacts

View File

@ -8,6 +8,9 @@ parameters:
- name: CtestRegex
type: string
default: .*
- name: Coverage
type: string
default: 'enabled'
stages:
@ -18,6 +21,7 @@ stages:
ServiceDirectory: ${{ parameters.ServiceDirectory }}
Artifacts: ${{ parameters.Artifacts }}
CtestRegex: ${{ parameters.CtestRegex }}
Coverage: ${{ parameters.Coverage }}
- ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:

View File

@ -17,6 +17,6 @@ steps:
displayName: cmake generate
- ${{ if eq(parameters.Build, true) }}:
- script: cmake ${{ parameters.BuildArgs }} --build .
- script: cmake --build . ${{ parameters.BuildArgs }}
workingDirectory: build
displayName: cmake build
displayName: cmake build

View File

@ -14,6 +14,7 @@
### Other changes and Improvements
- Add high-level and simplified core.hpp file for simpler include experience for customers.
- Add code coverage using gcov with gcc.
- Update SDK-defined exception types to be classes instead of structs.
### Bug Fixes

View File

@ -21,6 +21,9 @@ if(BUILD_TRANSPORT_WINHTTP)
SET(WIN_TRANSPORT_ADAPTER_SRC src/http/winhttp/win_http_transport.cpp)
endif()
# CodeCoverage will validate requirements
include(CodeCoverage)
add_library (
${TARGET_NAME}
src/context.cpp
@ -48,6 +51,9 @@ target_include_directories (${TARGET_NAME} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURR
# make sure that users can consume the project as a library.
add_library (Azure::Core ALIAS ${TARGET_NAME})
# coverage. Has no effect if BUILD_CODE_COVERAGE is OFF
create_code_coverage(core ${TARGET_NAME} "${TARGET_NAME}-test")
target_include_directories(${TARGET_NAME} PUBLIC ${CURL_INCLUDE_DIRS})
target_link_libraries(${TARGET_NAME} PRIVATE CURL::libcurl)

View File

@ -15,10 +15,18 @@ set (AZURE_IDENTITY_SOURCE
src/version.cpp
)
# Uncomment once identity have tests
# CodeCoverage will validate requirements
# include(CodeCoverage)
add_library(azure-identity ${AZURE_IDENTITY_HEADER} ${AZURE_IDENTITY_SOURCE})
target_include_directories(azure-identity PUBLIC inc)
target_link_libraries(azure-identity azure-core)
# Uncomment once identity have tests
# coverage. Has no effect if BUILD_CODE_COVERAGE is OFF
# create_code_coverage(identity azure-identity "azure-identity-test")
add_library(azure::identity ALIAS azure-identity)
get_az_version("${CMAKE_CURRENT_SOURCE_DIR}/inc/azure/identity/version.hpp")

View File

@ -32,6 +32,7 @@ stages:
parameters:
ServiceDirectory: identity
CtestRegex: azure-identity
Coverage: disabled
Artifacts:
- Name: azure-identity
Path: azure-identity

View File

@ -30,6 +30,9 @@ set (AZURE_STORAGE_BLOB_SOURCE
src/page_blob_client.cpp
)
# CodeCoverage will validate requirements
include(CodeCoverage)
add_library(azure-storage-blobs ${AZURE_STORAGE_BLOB_HEADER} ${AZURE_STORAGE_BLOB_SOURCE})
target_include_directories(azure-storage-blobs PUBLIC inc)
target_link_libraries(azure-storage-blobs azure::storage::common)
@ -37,6 +40,8 @@ target_link_libraries(azure-storage-blobs azure::storage::common)
get_az_version("${CMAKE_CURRENT_SOURCE_DIR}/inc/azure/storage/blobs/version.hpp")
add_library(azure::storage::blobs ALIAS azure-storage-blobs)
# coverage. Has no effect if BUILD_CODE_COVERAGE is OFF
create_code_coverage(storage azure-storage-blobs azure-storage-test)
target_sources(
azure-storage-test

View File

@ -36,6 +36,9 @@ set(AZURE_STORAGE_COMMON_SOURCE
src/xml_wrapper.cpp
)
# CodeCoverage will validate requirements
include(CodeCoverage)
add_library(azure-storage-common ${AZURE_STORAGE_COMMON_HEADER} ${AZURE_STORAGE_COMMON_SOURCE})
set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
@ -54,6 +57,8 @@ else()
target_link_libraries(azure-storage-common OpenSSL::SSL OpenSSL::Crypto)
endif()
# coverage. Has no effect if BUILD_CODE_COVERAGE is OFF
create_code_coverage(storage azure-storage-common azure-storage-test)
get_az_version("${CMAKE_CURRENT_SOURCE_DIR}/inc/azure/storage/common/version.hpp")
add_library(azure::storage::common ALIAS azure-storage-common)

View File

@ -29,10 +29,16 @@ set (AZURE_STORAGE_DATALAKE_SOURCE
src/datalake_utilities.cpp
)
# CodeCoverage will validate requirements
include(CodeCoverage)
add_library(azure-storage-files-datalake ${AZURE_STORAGE_DATALAKE_HEADER} ${AZURE_STORAGE_DATALAKE_SOURCE})
target_include_directories(azure-storage-files-datalake PUBLIC inc)
target_link_libraries(azure-storage-files-datalake azure-storage-blobs)
# coverage. Has no effect if BUILD_CODE_COVERAGE is OFF
create_code_coverage(storage azure-storage-files-datalake azure-storage-test)
get_az_version("${CMAKE_CURRENT_SOURCE_DIR}/inc/azure/storage/files/datalake/version.hpp")
add_library(azure::storage::files::datalake ALIAS azure-storage-files-datalake)

View File

@ -26,10 +26,16 @@ set (AZURE_STORAGE_SHARES_SOURCE
src/share_service_client.cpp
)
# CodeCoverage will validate requirements
include(CodeCoverage)
add_library(azure-storage-files-shares ${AZURE_STORAGE_SHARES_HEADER} ${AZURE_STORAGE_SHARES_SOURCE})
target_include_directories(azure-storage-files-shares PUBLIC inc)
target_link_libraries(azure-storage-files-shares azure-storage-common)
# coverage. Has no effect if BUILD_CODE_COVERAGE is OFF
create_code_coverage(storage azure-storage-files-shares azure-storage-test)
get_az_version("${CMAKE_CURRENT_SOURCE_DIR}/inc/azure/storage/files/shares/version.hpp")
add_library(azure::storage::files::shares ALIAS azure-storage-files-shares)

View File

@ -37,6 +37,7 @@ stages:
ServiceDirectory: storage
# TODO: Change to azure-storage once we have an strategy to run livetests or use test recordings
CtestRegex: azure-core
Coverage: disabled
Artifacts:
- Name: azure-storage-common
Path: azure-storage-common

View File

@ -8,6 +8,9 @@ project(azure-template LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED True)
# CodeCoverage will validate requirements
include(CodeCoverage)
add_library (
azure-template
src/version.cpp
@ -16,6 +19,9 @@ add_library (
target_include_directories (azure-template PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/inc>)
# coverage. Has no effect if BUILD_CODE_COVERAGE is OFF
create_code_coverage(template azure-template azure-template-test)
# make sure that users can consume the project as a library.
add_library (Azure::Template ALIAS azure-template)