From ba9a6bd5b3e36d9ac78268498fe85216ada24c45 Mon Sep 17 00:00:00 2001
From: Tim Ramlot <42113979+inteon@users.noreply.github.com>
Date: Thu, 4 Aug 2022 11:15:18 +0000
Subject: [PATCH] add pruning logic for gotestsum junit xml output
Signed-off-by: Tim Ramlot <42113979+inteon@users.noreply.github.com>
---
hack/prune-junit-xml/prunexml.go | 235 ++++++++++++++++++++++++++
hack/prune-junit-xml/prunexml_test.go | 102 +++++++++++
make/test.mk | 17 +-
3 files changed, 346 insertions(+), 8 deletions(-)
create mode 100644 hack/prune-junit-xml/prunexml.go
create mode 100644 hack/prune-junit-xml/prunexml_test.go
diff --git a/hack/prune-junit-xml/prunexml.go b/hack/prune-junit-xml/prunexml.go
new file mode 100644
index 000000000..a9d0b9108
--- /dev/null
+++ b/hack/prune-junit-xml/prunexml.go
@@ -0,0 +1,235 @@
+/*
+Copyright 2022 The cert-manager Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+Copyright 2022 The Kubernetes Authors.
+Started from https://github.com/kubernetes/kubernetes/blob/978d9683f5c253cf62225dc0656c0bfeb2a7d339/cmd/prune-junit-xml/prunexml.go
+*/
+
+// This tool prevents the junit xml files from becoming too big.
+// Because big files cause longer loading times and might break
+// some tools. It also improves the web ui experience by reducing
+// visual noise.
+// More info: https://github.com/kubernetes/kubernetes/pull/109112
+//
+// The following processing steps are included:
+// - compacting all fuzz tests (often 1000+) into a single entry
+// - removing empty testsuites
+// - clipping the output from failures or skips
+
+package main
+
+import (
+ "encoding/xml"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "regexp"
+ "strconv"
+)
+
+// JUnitTestSuites is a collection of JUnit test suites.
+type JUnitTestSuites struct {
+ XMLName xml.Name `xml:"testsuites"`
+ Suites []JUnitTestSuite `xml:"testsuite,omitempty"`
+}
+
+// JUnitTestSuite is a single JUnit test suite which may contain many
+// testcases.
+type JUnitTestSuite struct {
+ XMLName xml.Name `xml:"testsuite"`
+ Tests int `xml:"tests,attr"`
+ Failures int `xml:"failures,attr"`
+ Time string `xml:"time,attr"`
+ Name string `xml:"name,attr"`
+ Properties []JUnitProperty `xml:"properties>property,omitempty"`
+ TestCases []JUnitTestCase `xml:"testcase,omitempty"`
+ Timestamp string `xml:"timestamp,attr"`
+}
+
+// JUnitTestCase is a single test case with its result.
+type JUnitTestCase struct {
+ XMLName xml.Name `xml:"testcase"`
+ Classname string `xml:"classname,attr"`
+ Name string `xml:"name,attr"`
+ Time string `xml:"time,attr"`
+ SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"`
+ Failure *JUnitFailure `xml:"failure,omitempty"`
+}
+
+// JUnitSkipMessage contains the reason why a testcase was skipped.
+type JUnitSkipMessage struct {
+ Message string `xml:"message,attr"`
+}
+
+// JUnitProperty represents a key/value pair used to define properties.
+type JUnitProperty struct {
+ Name string `xml:"name,attr"`
+ Value string `xml:"value,attr"`
+}
+
+// JUnitFailure contains data related to a failed test.
+type JUnitFailure struct {
+ Message string `xml:"message,attr"`
+ Type string `xml:"type,attr"`
+ Contents string `xml:",chardata"`
+}
+
+var fuzzNameRegex = regexp.MustCompile(`^(.*)\/fuzz_\d+$`)
+
+func main() {
+ maxTextSize := flag.Int("max-text-size", 1, "maximum size of attribute or text (in MB)")
+ flag.Parse()
+
+ if flag.NArg() > 0 {
+ for _, path := range flag.Args() {
+ fmt.Printf("processing junit xml file : %s\n", path)
+ xmlReader, err := os.Open(path)
+ if err != nil {
+ panic(err)
+ }
+ defer xmlReader.Close()
+ suites, err := fetchXML(xmlReader) // convert MB into bytes (roughly!)
+ if err != nil {
+ panic(err)
+ }
+
+ pruneXML(suites, *maxTextSize*1e6) // convert MB into bytes (roughly!)
+
+ xmlWriter, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+ if err != nil {
+ panic(err)
+ }
+ defer xmlWriter.Close()
+ err = streamXML(xmlWriter, suites)
+ if err != nil {
+ panic(err)
+ }
+ fmt.Println("done.")
+ }
+ }
+}
+
+func pruneXML(suites *JUnitTestSuites, maxBytes int) {
+ // filter empty testSuites
+ filteredSuites := []JUnitTestSuite{}
+ for _, suite := range suites.Suites {
+ if suite.Tests+suite.Failures+len(suite.TestCases) > 0 {
+ filteredSuites = append(filteredSuites, suite)
+ }
+ }
+ suites.Suites = filteredSuites
+
+ // compact fuzz tests
+ compactedSuites := []JUnitTestSuite{}
+ for _, suite := range suites.Suites {
+ filteredTestCases := []*JUnitTestCase{}
+ fuzzTestCases := map[string]*JUnitTestCase{}
+ for _, testcase := range suite.TestCases {
+ testcase := testcase
+ matches := fuzzNameRegex.FindStringSubmatch(testcase.Name)
+ if len(matches) > 1 {
+ if ftc, ok := fuzzTestCases[matches[1]]; ok {
+ if testcase.Failure != nil {
+ ftc.Failure = testcase.Failure // we only display one failure
+ ftc.SkipMessage = nil
+ }
+ if testcase.SkipMessage != nil && ftc.Failure == nil {
+ ftc.SkipMessage = testcase.SkipMessage // only display SkipMessage if no other fuzz has failed
+ }
+ ftc.Time = incrementTime(ftc.Time, testcase.Time)
+ } else {
+ testcase.Name = matches[1] + "/fuzz_xxxx"
+ fuzzTestCases[matches[1]] = &testcase
+ filteredTestCases = append(filteredTestCases, &testcase)
+ }
+ } else {
+ filteredTestCases = append(filteredTestCases, &testcase)
+ }
+ }
+
+ suite.TestCases = []JUnitTestCase{}
+ suite.Tests = 0
+ suite.Failures = 0
+ for _, testcase := range filteredTestCases {
+ suite.TestCases = append(suite.TestCases, *testcase)
+ suite.Tests += 1
+ if testcase.Failure != nil {
+ suite.Failures += 1
+ }
+
+ }
+ compactedSuites = append(compactedSuites, suite)
+ }
+ suites.Suites = compactedSuites
+
+ // clip output messages
+ for _, suite := range suites.Suites {
+ for _, testcase := range suite.TestCases {
+ if testcase.SkipMessage != nil {
+ if len(testcase.SkipMessage.Message) > maxBytes {
+ fmt.Printf("clipping skip message in test case : %s\n", testcase.Name)
+ testcase.SkipMessage.Message = "[... clipped...]" +
+ testcase.SkipMessage.Message[len(testcase.SkipMessage.Message)-maxBytes:]
+ }
+ }
+ if testcase.Failure != nil {
+ if len(testcase.Failure.Contents) > maxBytes {
+ fmt.Printf("clipping failure message in test case : %s\n", testcase.Name)
+ testcase.Failure.Contents = "[... clipped...]" +
+ testcase.Failure.Contents[len(testcase.Failure.Contents)-maxBytes:]
+ }
+ }
+ }
+ }
+}
+
+func incrementTime(total string, delta string) string {
+ totalTime, err := strconv.ParseFloat(total, 32)
+ if err != nil {
+ return total
+ }
+ deltaTime, err := strconv.ParseFloat(delta, 32)
+ if err != nil {
+ return total
+ }
+ return fmt.Sprintf("%.6f", totalTime+deltaTime)
+}
+
+func fetchXML(xmlReader io.Reader) (*JUnitTestSuites, error) {
+ decoder := xml.NewDecoder(xmlReader)
+ var suites JUnitTestSuites
+ err := decoder.Decode(&suites)
+ if err != nil {
+ return nil, err
+ }
+ return &suites, nil
+}
+
+func streamXML(writer io.Writer, in *JUnitTestSuites) error {
+ _, err := writer.Write([]byte("\n"))
+ if err != nil {
+ return err
+ }
+ encoder := xml.NewEncoder(writer)
+ encoder.Indent("", "\t")
+ err = encoder.Encode(in)
+ if err != nil {
+ return err
+ }
+ return encoder.Flush()
+}
diff --git a/hack/prune-junit-xml/prunexml_test.go b/hack/prune-junit-xml/prunexml_test.go
new file mode 100644
index 000000000..4b7e49d3b
--- /dev/null
+++ b/hack/prune-junit-xml/prunexml_test.go
@@ -0,0 +1,102 @@
+/*
+Copyright 2022 The cert-manager Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+Copyright 2022 The Kubernetes Authors.
+Started from https://github.com/kubernetes/kubernetes/blob/978d9683f5c253cf62225dc0656c0bfeb2a7d339/cmd/prune-junit-xml/prunexml_test.go
+*/
+
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestPruneXML(t *testing.T) {
+ sourceXML := `
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x147
k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0e5f8edb0, {0xc0efe16f88?, 0xc1169d3a88?, 0x1804787?})
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x32
io.ReadAtLeast({0x55c5720, 0xc0e5f8edb0}, {0xc0efe16f88, 0x5, 0x5}, 0x5)
/usr/local/go/src/io/io.go:331 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:350
k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0f3cd67e0, {0xc0efe16f88, 0x5, 0x5})
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xa5
k8s.io/kubernetes/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc0efe16f78, 0x7fffffff)
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47
k8s.io/kubernetes/vendor/google.golang.org/grpc.recvAndDecompress(0xc1169d3c58?, 0xc0f3cd67e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0})
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66
k8s.io/kubernetes/vendor/google.golang.org/grpc.recv(0x172b28f?, {0x7f837c291d58, 0x7f84350}, 0x6f5a274d6e8f284c?, {0x0?, 0x0?}, {0x4be7d40, 0xc0f8c01d50}, 0x0?, 0x0, ...)
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/rpc_util.go:758 +0x6e
k8s.io/kubernetes/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc0eb72d800, {0x4be7d40?, 0xc0f8c01d50}, 0x2?)
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:970 +0x2b0
k8s.io/kubernetes/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4be7d40?)
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:821 +0x25
k8s.io/kubernetes/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc0f3cd65a0, 0xc1169d3e78, 0xc1169d3e48)
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:675 +0x2f6
k8s.io/kubernetes/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc0f3cd65a0, {0x4be7d40?, 0xc0f8c01d50?})
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:820 +0x11f
k8s.io/kubernetes/vendor/github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg(0xc0efe16f90, {0x4be7d40?, 0xc0f8c01d50?})
/home/prow/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_metrics.go:160
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+`
+
+ outputXML := `
+
+
+
+
+
+
+
+
+
+
+ [... clipped...]prometheus/client_metrics.go:160
+
+
+
+
+
+
+
+
+
+`
+ suites, _ := fetchXML(strings.NewReader(sourceXML))
+ pruneXML(suites, 32)
+ var output bytes.Buffer
+ writer := bufio.NewWriter(&output)
+ _ = streamXML(writer, suites)
+ _ = writer.Flush()
+ assert.Equal(t, outputXML, string(output.Bytes()), "xml was not pruned correctly")
+}
diff --git a/make/test.mk b/make/test.mk
index 46e77f4eb..5a010a19f 100644
--- a/make/test.mk
+++ b/make/test.mk
@@ -3,7 +3,7 @@ export KUBEBUILDER_ASSETS=$(PWD)/$(BINDIR)/tools
# WHAT can be used to control which unit tests are run by "make test"; defaults to running all
# tests except e2e tests (which require more significant setup)
# For example: make WHAT=./pkg/util/pki test-pretty to only run the PKI utils tests
-WHAT ?= ./pkg/... ./cmd/... ./internal/... ./test/...
+WHAT ?= ./pkg/... ./cmd/... ./internal/... ./test/... ./hack/prune-junit-xml/...
.PHONY: test
## Test is the workhorse test command which by default runs all unit and
@@ -24,14 +24,15 @@ test: setup-integration-tests $(BINDIR)/tools/gotestsum $(BINDIR)/tools/etcd $(B
## issues with dashboards and UIs.
##
## @category CI
-test-ci: setup-integration-tests $(BINDIR)/tools/gotestsum $(BINDIR)/tools/etcd $(BINDIR)/tools/kubectl $(BINDIR)/tools/kube-apiserver
- @# Fuzz tests are hidden from JUnit output because they can break dashboards.
- @# They look like this:
- @#
+test-ci: setup-integration-tests $(BINDIR)/tools/gotestsum $(BINDIR)/tools/etcd $(BINDIR)/tools/kubectl $(BINDIR)/tools/kube-apiserver $(DEPENDS_ON_GO)
@mkdir -p $(ARTIFACTS)
- $(GOTESTSUM) --junitfile $(ARTIFACTS)/junit_make-test-ci.xml \
- --post-run-command $$'bash -c "awk \'$$1 \!~ /\\/fuzz_\\d+// { print $$2 }\' - $$GOTESTSUM_JUNITFILE >/tmp/$$$$ && mv /tmp/$$$$ $$GOTESTSUM_JUNITFILE"' \
- --junitfile-testsuite-name short --junitfile-testcase-classname relative -- $(WHAT)
+ $(GOTESTSUM) \
+ --junitfile $(ARTIFACTS)/junit_make-test-ci.xml \
+ --junitfile-testsuite-name short \
+ --junitfile-testcase-classname relative \
+ --post-run-command $$'bash -c "$(GO) run hack/prune-junit-xml/prunexml.go $$GOTESTSUM_JUNITFILE"' \
+ -- \
+ $(WHAT)
.PHONY: unit-test
## Same as `test` but only runs the unit tests. By "unit tests", we mean tests