Add coverage enforcement (#2992)

* Add coverage enforcement

* Add spelling exception for mspremier

* Use previous build coverage data, enforce 3% variance, use Skip.CoverageEnforcement feature flag

* Use version 8 of quality checks

* Remove forceCoverageImprovement, it doesn't allow 0% improvements (e.g. doc changes, etc.)

* branch -> branches

* Configure coverage thresholds per-service

* Use fixed coverage

* Review feedback

* Per-service coverage plumbing

* Flip branch and line coverage requirements for KV to accurately reflect current baseline

.vscode/cspell.json

@@ -116,7 +116,8 @@
       "filename": "**/eng/pipelines/templates/**/*.yml",
       "words": [
         "lcov",
-        "iname"
+        "iname",
+        "mspremier"
       ]
     },
     {

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -20,6 +20,12 @@ parameters:
   - name: CoverageReportPath
     type: string
     default: 'sdk/*/*/*cov_xml.xml'
+  - name: LineCoverageTarget
+    type: number
+    default: 95
+  - name: BranchCoverageTarget
+    type: number
+    default: 70
 
 jobs:
   - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
@@ -37,6 +43,8 @@ jobs:
         CtestRegex: ${{ parameters.CtestRegex }}
         CoverageReportPath: ${{ parameters.CoverageReportPath }}
         CoverageEnabled: ${{ parameters.CoverageEnabled }}
+        LineCoverageTarget: ${{ parameters.LineCoverageTarget }}
+        BranchCoverageTarget: ${{ parameters.BranchCoverageTarget }}
 
   # Disable build for cpp - client
   - ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}:

eng/pipelines/templates/jobs/ci.tests.yml

@@ -19,6 +19,13 @@ parameters:
     default: true
   - name: Matrix
     type: string
+  - name: LineCoverageTarget
+    type: number
+    default: 95
+  - name: BranchCoverageTarget
+    type: number
+    default: 70
+
 
   # Supplied by archetype-sdk-tests-generate.yml dynamic matrix generator
   - name: CloudConfig
@@ -147,3 +154,23 @@ jobs:
             summaryFileLocation: '$(Build.SourcesDirectory)/**/Cobertura.xml'
           displayName: Publish Code Coverage to DevOps
           condition: and(succeededOrFailed(), eq(variables['CODE_COVERAGE'], 'enabled'))
+
+        - task: mspremier.BuildQualityChecks.QualityChecks-task.BuildQualityChecks@8
+          displayName: Check line coverage
+          inputs:
+            checkCoverage: true
+            coverageFailOption: fixed
+            coverageType: line
+            # Minimum baseline for line coverage
+            coverageThreshold: ${{ parameters.LineCoverageTarget }}
+          condition: and(succeededOrFailed(), eq(variables['CODE_COVERAGE'], 'enabled'), eq(variables['Skip.LineCoverageEnforcement'], ''))
+
+        - task: mspremier.BuildQualityChecks.QualityChecks-task.BuildQualityChecks@8
+          displayName: Check branch coverage
+          inputs:
+            checkCoverage: true
+            coverageFailOption: fixed
+            coverageType: branches
+            # Minimum baseline for branch coverage
+            coverageThreshold: ${{ parameters.BranchCoverageTarget }}
+          condition: and(succeededOrFailed(), eq(variables['CODE_COVERAGE'], 'enabled'), eq(variables['Skip.BranchCoverageEnforcement'], ''))

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -26,6 +26,12 @@ parameters:
 - name: LiveTestTimeoutInMinutes
   type: number
   default: 60
+- name: LineCoverageTarget
+  type: number
+  default: 95
+- name: BranchCoverageTarget
+  type: number
+  default: 70
 
 stages:
   - stage: Build
@@ -37,6 +43,8 @@ stages:
           CtestRegex: ${{ parameters.CtestRegex }}
           CoverageEnabled: ${{ parameters.CoverageEnabled }}
           CoverageReportPath: ${{ parameters.CoverageReportPath }}
+          LineCoverageTarget: ${{ parameters.LineCoverageTarget }}
+          BranchCoverageTarget: ${{ parameters.BranchCoverageTarget }}
           ${{ if eq(parameters.ServiceDirectory, 'template') }}:
             TestPipeline: true
 

sdk/core/ci.yml

@@ -37,6 +37,8 @@ stages:
       CtestRegex: azure-core.
       LiveTestCtestRegex: '"azure-core.|json-test"'
       LiveTestTimeoutInMinutes: 90 # default is 60 min. We need a little longer on worst case for Win+jsonTests
+      LineCoverageTarget: 79
+      BranchCoverageTarget: 30
       Artifacts:
         - Name: azure-core
           Path: azure-core

sdk/identity/ci.yml

@@ -33,6 +33,8 @@ stages:
       ServiceDirectory: identity
       CtestRegex: azure-identity-unittest.
       LiveTestCtestRegex: azure-identity-livetest.
+      LineCoverageTarget: 82
+      BranchCoverageTarget: 32
       Artifacts:
         - Name: azure-identity
           Path: azure-identity

sdk/keyvault/ci.yml

@@ -37,6 +37,8 @@ stages:
       LiveTestCtestRegex: "azure-security-keyvault.*"
       LiveTestTimeoutInMinutes: 120
       SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+      LineCoverageTarget: 37
+      BranchCoverageTarget: 15
       Artifacts:
         - Name: azure-security-keyvault-keys
           Path: azure-security-keyvault-keys

sdk/template/ci.yml

@@ -34,6 +34,13 @@ stages:
     parameters:
       ServiceDirectory: template
       CtestRegex: azure-template
+      # The azure-template project has lower branch coverage than the default.
+      # This overrides the branch coverage floor until the issue can be
+      # addressed. Once it is addressed this line should be removed so that
+      # projects generated from this ci.yml file don't set a coverage target
+      # that is lower than the baseline defined in archetype-sdk-client.yml
+      # Issue: https://github.com/Azure/azure-sdk-for-cpp/issues/3022
+      BranchCoverageTarget: 33
       Artifacts:
         - Name: azure-template
           Path: azure-template