add pruning logic for gotestsum junit xml output

Signed-off-by: Tim Ramlot <42113979+inteon@users.noreply.github.com>
This commit is contained in:
Tim Ramlot 2022-08-04 11:15:18 +00:00
parent 12342d88e5
commit ba9a6bd5b3
3 changed files with 346 additions and 8 deletions

View File

@ -0,0 +1,235 @@
/*
Copyright 2022 The cert-manager Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Copyright 2022 The Kubernetes Authors.
Started from https://github.com/kubernetes/kubernetes/blob/978d9683f5c253cf62225dc0656c0bfeb2a7d339/cmd/prune-junit-xml/prunexml.go
*/
// This tool prevents the junit xml files from becoming too big.
// Because big files cause longer loading times and might break
// some tools. It also improves the web ui experience by reducing
// visual noise.
// More info: https://github.com/kubernetes/kubernetes/pull/109112
//
// The following processing steps are included:
// - compacting all fuzz tests (often 1000+) into a single entry
// - removing empty testsuites
// - clipping the output from failures or skips
package main
import (
"encoding/xml"
"flag"
"fmt"
"io"
"os"
"regexp"
"strconv"
)
// JUnitTestSuites is a collection of JUnit test suites.
type JUnitTestSuites struct {
XMLName xml.Name `xml:"testsuites"`
Suites []JUnitTestSuite `xml:"testsuite,omitempty"`
}
// JUnitTestSuite is a single JUnit test suite which may contain many
// testcases.
type JUnitTestSuite struct {
XMLName xml.Name `xml:"testsuite"`
Tests int `xml:"tests,attr"`
Failures int `xml:"failures,attr"`
Time string `xml:"time,attr"`
Name string `xml:"name,attr"`
Properties []JUnitProperty `xml:"properties>property,omitempty"`
TestCases []JUnitTestCase `xml:"testcase,omitempty"`
Timestamp string `xml:"timestamp,attr"`
}
// JUnitTestCase is a single test case with its result.
type JUnitTestCase struct {
XMLName xml.Name `xml:"testcase"`
Classname string `xml:"classname,attr"`
Name string `xml:"name,attr"`
Time string `xml:"time,attr"`
SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"`
Failure *JUnitFailure `xml:"failure,omitempty"`
}
// JUnitSkipMessage contains the reason why a testcase was skipped.
type JUnitSkipMessage struct {
Message string `xml:"message,attr"`
}
// JUnitProperty represents a key/value pair used to define properties.
type JUnitProperty struct {
Name string `xml:"name,attr"`
Value string `xml:"value,attr"`
}
// JUnitFailure contains data related to a failed test.
type JUnitFailure struct {
Message string `xml:"message,attr"`
Type string `xml:"type,attr"`
Contents string `xml:",chardata"`
}
var fuzzNameRegex = regexp.MustCompile(`^(.*)\/fuzz_\d+$`)
func main() {
maxTextSize := flag.Int("max-text-size", 1, "maximum size of attribute or text (in MB)")
flag.Parse()
if flag.NArg() > 0 {
for _, path := range flag.Args() {
fmt.Printf("processing junit xml file : %s\n", path)
xmlReader, err := os.Open(path)
if err != nil {
panic(err)
}
defer xmlReader.Close()
suites, err := fetchXML(xmlReader) // convert MB into bytes (roughly!)
if err != nil {
panic(err)
}
pruneXML(suites, *maxTextSize*1e6) // convert MB into bytes (roughly!)
xmlWriter, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
panic(err)
}
defer xmlWriter.Close()
err = streamXML(xmlWriter, suites)
if err != nil {
panic(err)
}
fmt.Println("done.")
}
}
}
func pruneXML(suites *JUnitTestSuites, maxBytes int) {
// filter empty testSuites
filteredSuites := []JUnitTestSuite{}
for _, suite := range suites.Suites {
if suite.Tests+suite.Failures+len(suite.TestCases) > 0 {
filteredSuites = append(filteredSuites, suite)
}
}
suites.Suites = filteredSuites
// compact fuzz tests
compactedSuites := []JUnitTestSuite{}
for _, suite := range suites.Suites {
filteredTestCases := []*JUnitTestCase{}
fuzzTestCases := map[string]*JUnitTestCase{}
for _, testcase := range suite.TestCases {
testcase := testcase
matches := fuzzNameRegex.FindStringSubmatch(testcase.Name)
if len(matches) > 1 {
if ftc, ok := fuzzTestCases[matches[1]]; ok {
if testcase.Failure != nil {
ftc.Failure = testcase.Failure // we only display one failure
ftc.SkipMessage = nil
}
if testcase.SkipMessage != nil && ftc.Failure == nil {
ftc.SkipMessage = testcase.SkipMessage // only display SkipMessage if no other fuzz has failed
}
ftc.Time = incrementTime(ftc.Time, testcase.Time)
} else {
testcase.Name = matches[1] + "/fuzz_xxxx"
fuzzTestCases[matches[1]] = &testcase
filteredTestCases = append(filteredTestCases, &testcase)
}
} else {
filteredTestCases = append(filteredTestCases, &testcase)
}
}
suite.TestCases = []JUnitTestCase{}
suite.Tests = 0
suite.Failures = 0
for _, testcase := range filteredTestCases {
suite.TestCases = append(suite.TestCases, *testcase)
suite.Tests += 1
if testcase.Failure != nil {
suite.Failures += 1
}
}
compactedSuites = append(compactedSuites, suite)
}
suites.Suites = compactedSuites
// clip output messages
for _, suite := range suites.Suites {
for _, testcase := range suite.TestCases {
if testcase.SkipMessage != nil {
if len(testcase.SkipMessage.Message) > maxBytes {
fmt.Printf("clipping skip message in test case : %s\n", testcase.Name)
testcase.SkipMessage.Message = "[... clipped...]" +
testcase.SkipMessage.Message[len(testcase.SkipMessage.Message)-maxBytes:]
}
}
if testcase.Failure != nil {
if len(testcase.Failure.Contents) > maxBytes {
fmt.Printf("clipping failure message in test case : %s\n", testcase.Name)
testcase.Failure.Contents = "[... clipped...]" +
testcase.Failure.Contents[len(testcase.Failure.Contents)-maxBytes:]
}
}
}
}
}
func incrementTime(total string, delta string) string {
totalTime, err := strconv.ParseFloat(total, 32)
if err != nil {
return total
}
deltaTime, err := strconv.ParseFloat(delta, 32)
if err != nil {
return total
}
return fmt.Sprintf("%.6f", totalTime+deltaTime)
}
func fetchXML(xmlReader io.Reader) (*JUnitTestSuites, error) {
decoder := xml.NewDecoder(xmlReader)
var suites JUnitTestSuites
err := decoder.Decode(&suites)
if err != nil {
return nil, err
}
return &suites, nil
}
func streamXML(writer io.Writer, in *JUnitTestSuites) error {
_, err := writer.Write([]byte("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"))
if err != nil {
return err
}
encoder := xml.NewEncoder(writer)
encoder.Indent("", "\t")
err = encoder.Encode(in)
if err != nil {
return err
}
return encoder.Flush()
}

View File

@ -0,0 +1,102 @@
/*
Copyright 2022 The cert-manager Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Copyright 2022 The Kubernetes Authors.
Started from https://github.com/kubernetes/kubernetes/blob/978d9683f5c253cf62225dc0656c0bfeb2a7d339/cmd/prune-junit-xml/prunexml_test.go
*/
package main
import (
"bufio"
"bytes"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestPruneXML(t *testing.T) {
sourceXML := `<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite tests="0" failures="0" time="0.000000" name="v1beta1" timestamp="2022-08-04T07:44:08Z">
<properties>
<property name="go.version" value="go1.19 linux/amd64"></property>
</properties>
</testsuite>
<testsuite tests="3" failures="1" time="271.610000" name="k8s.io/kubernetes/test/integration/apiserver" timestamp="">
<properties>
<property name="go.version" value="go1.18 linux/amd64"></property>
</properties>
<testcase classname="k8s.io/kubernetes/test/integration/apimachinery" name="TestWatchRestartsIfTimeoutNotReached/group/InformerWatcher_survives_closed_watches" time="30.050000"></testcase>
<testcase classname="k8s.io/kubernetes/test/integration/apiserver" name="TestMaxResourceSize/JSONPatchType_should_handle_a_patch_just_under_the_max_limit" time="0.000000">
<skipped message="=== RUN TestMaxResourceSize/JSONPatchType_should_handle_a_patch_just_under_the_max_limit&#xA; max_request_body_bytes_test.go:89: skipping expensive test&#xA; --- SKIP: TestMaxResourceSize/JSONPatchType_should_handle_a_patch_just_under_the_max_limit (0.00s)&#xA;"></skipped>
</testcase>
<testcase classname="k8s.io/kubernetes/test/integration/apimachinery" name="TestSchedulerInformers" time="-0.000000">
<failure message="Failed" type="">&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x147&#xA;k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0e5f8edb0, {0xc0efe16f88?, 0xc1169d3a88?, 0x1804787?})&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x32&#xA;io.ReadAtLeast({0x55c5720, 0xc0e5f8edb0}, {0xc0efe16f88, 0x5, 0x5}, 0x5)&#xA;&#x9;/usr/local/go/src/io/io.go:331 +0x9a&#xA;io.ReadFull(...)&#xA;&#x9;/usr/local/go/src/io/io.go:350&#xA;k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0f3cd67e0, {0xc0efe16f88, 0x5, 0x5})&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xa5&#xA;k8s.io/kubernetes/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc0efe16f78, 0x7fffffff)&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47&#xA;k8s.io/kubernetes/vendor/google.golang.org/grpc.recvAndDecompress(0xc1169d3c58?, 0xc0f3cd67e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0})&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66&#xA;k8s.io/kubernetes/vendor/google.golang.org/grpc.recv(0x172b28f?, {0x7f837c291d58, 0x7f84350}, 0x6f5a274d6e8f284c?, {0x0?, 0x0?}, {0x4be7d40, 0xc0f8c01d50}, 0x0?, 0x0, ...)&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/rpc_util.go:758 +0x6e&#xA;k8s.io/kubernetes/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc0eb72d800, {0x4be7d40?, 0xc0f8c01d50}, 0x2?)&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:970 +0x2b0&#xA;k8s.io/kubernetes/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4be7d40?)&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:821 +0x25&#xA;k8s.io/kubernetes/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc0f3cd65a0, 0xc1169d3e78, 0xc1169d3e48)&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:675 +0x2f6&#xA;k8s.io/kubernetes/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc0f3cd65a0, {0x4be7d40?, 0xc0f8c01d50?})&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:820 +0x11f&#xA;k8s.io/kubernetes/vendor/github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg(0xc0efe16f90, {0x4be7d40?, 0xc0f8c01d50?})&#xA;&#x9;/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_metrics.go:160</failure>
</testcase>
</testsuite>
<testsuite tests="20002" failures="0" time="1.212000" name="certificaterequests" timestamp="2022-08-04T07:44:08Z">
<properties>
<property name="go.version" value="go1.19 linux/amd64"></property>
</properties>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_0" time="0.010000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_2" time="0.010000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_1" time="0.000000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_4" time="0.000000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_5" time="0.000000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_3" time="0.000000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_6" time="0.000000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_7" time="0.000000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_8" time="0.000000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_11" time="0.000000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_10" time="0.000000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_13" time="0.000000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApplyStatus" time="1.610000"></testcase>
</testsuite>
</testsuites>`
outputXML := `<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite tests="3" failures="1" time="271.610000" name="k8s.io/kubernetes/test/integration/apiserver" timestamp="">
<properties>
<property name="go.version" value="go1.18 linux/amd64"></property>
</properties>
<testcase classname="k8s.io/kubernetes/test/integration/apimachinery" name="TestWatchRestartsIfTimeoutNotReached/group/InformerWatcher_survives_closed_watches" time="30.050000"></testcase>
<testcase classname="k8s.io/kubernetes/test/integration/apiserver" name="TestMaxResourceSize/JSONPatchType_should_handle_a_patch_just_under_the_max_limit" time="0.000000">
<skipped message="[... clipped...]ust_under_the_max_limit (0.00s)&#xA;"></skipped>
</testcase>
<testcase classname="k8s.io/kubernetes/test/integration/apimachinery" name="TestSchedulerInformers" time="-0.000000">
<failure message="Failed" type="">[... clipped...]prometheus/client_metrics.go:160</failure>
</testcase>
</testsuite>
<testsuite tests="2" failures="0" time="1.212000" name="certificaterequests" timestamp="2022-08-04T07:44:08Z">
<properties>
<property name="go.version" value="go1.19 linux/amd64"></property>
</properties>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApply/fuzz_xxxx" time="0.020000"></testcase>
<testcase classname="internal/controller/certificaterequests" name="Test_serializeApplyStatus" time="1.610000"></testcase>
</testsuite>
</testsuites>`
suites, _ := fetchXML(strings.NewReader(sourceXML))
pruneXML(suites, 32)
var output bytes.Buffer
writer := bufio.NewWriter(&output)
_ = streamXML(writer, suites)
_ = writer.Flush()
assert.Equal(t, outputXML, string(output.Bytes()), "xml was not pruned correctly")
}

View File

@ -3,7 +3,7 @@ export KUBEBUILDER_ASSETS=$(PWD)/$(BINDIR)/tools
# WHAT can be used to control which unit tests are run by "make test"; defaults to running all
# tests except e2e tests (which require more significant setup)
# For example: make WHAT=./pkg/util/pki test-pretty to only run the PKI utils tests
WHAT ?= ./pkg/... ./cmd/... ./internal/... ./test/...
WHAT ?= ./pkg/... ./cmd/... ./internal/... ./test/... ./hack/prune-junit-xml/...
.PHONY: test
## Test is the workhorse test command which by default runs all unit and
@ -24,14 +24,15 @@ test: setup-integration-tests $(BINDIR)/tools/gotestsum $(BINDIR)/tools/etcd $(B
## issues with dashboards and UIs.
##
## @category CI
test-ci: setup-integration-tests $(BINDIR)/tools/gotestsum $(BINDIR)/tools/etcd $(BINDIR)/tools/kubectl $(BINDIR)/tools/kube-apiserver
@# Fuzz tests are hidden from JUnit output because they can break dashboards.
@# They look like this:
@# <testcase classname="internal/controller/certificates" name="Test_serializeApplyStatus/fuzz_8358"></testcase>
test-ci: setup-integration-tests $(BINDIR)/tools/gotestsum $(BINDIR)/tools/etcd $(BINDIR)/tools/kubectl $(BINDIR)/tools/kube-apiserver $(DEPENDS_ON_GO)
@mkdir -p $(ARTIFACTS)
$(GOTESTSUM) --junitfile $(ARTIFACTS)/junit_make-test-ci.xml \
--post-run-command $$'bash -c "awk \'$$1 \!~ /\\/fuzz_\\d+// { print $$2 }\' - $$GOTESTSUM_JUNITFILE >/tmp/$$$$ && mv /tmp/$$$$ $$GOTESTSUM_JUNITFILE"' \
--junitfile-testsuite-name short --junitfile-testcase-classname relative -- $(WHAT)
$(GOTESTSUM) \
--junitfile $(ARTIFACTS)/junit_make-test-ci.xml \
--junitfile-testsuite-name short \
--junitfile-testcase-classname relative \
--post-run-command $$'bash -c "$(GO) run hack/prune-junit-xml/prunexml.go $$GOTESTSUM_JUNITFILE"' \
-- \
$(WHAT)
.PHONY: unit-test
## Same as `test` but only runs the unit tests. By "unit tests", we mean tests