7350863d8a
* This is to ensure Vault conformance passes since it outputs SANs in different order to other issuers * Matcher was tested manually only we will add tests to it in future Signed-off-by: SpectralHiss <houssem.elfekih@jetstack.io>
212 lines
8.6 KiB
Bash
Executable File
212 lines
8.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Copyright 2022 The cert-manager Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# shellcheck disable=SC2059
|
|
|
|
here=$(dirname "${BASH_SOURCE[0]}")
|
|
source "$here/config/lib.sh"
|
|
cd "$here/.." || exit 1
|
|
set -e
|
|
|
|
_default_bindir=$(make print-bindir)
|
|
|
|
BINDIR=${BINDIR:-$_default_bindir}
|
|
|
|
# Why do we only run 20 tests concurrently? Because we have noticed that
|
|
# many tests start timing out when the Prow pod gets overloaded. We are
|
|
# using a n1-standard-8 VM (7900m vCPU and 24GB RAM), and the pod requests
|
|
# 3500m of vCPU and 12GB of RAM.
|
|
#
|
|
# The components that seem to overload are kyverno (which is in the hot
|
|
# path of kube-apiserver), etcd, the kube-apiserver. cert-manager then
|
|
# becomes sluggish due to slow calls to the apiserver.
|
|
#
|
|
# In the following table, the first column shows the various -nodes values
|
|
# tested when running ginkgo. The "test duration" is the time spent while
|
|
# running "ginkgo", and the column"timeouts" column shows the number of
|
|
# tests that failed with a time out (including the tests that are retried;
|
|
# tests that show in the "Flaky" column in the Prow UI are thus counted
|
|
# twice).
|
|
#
|
|
#
|
|
# | nodes | ginkgo duration | timeouts | total duration | startup time | link |
|
|
# |-------|-----------------|----------|----------------|----------------|-------|
|
|
# | 5 | 37m 49s | 0 | 40m 21s | 2m 32s (hot) | [1][] |
|
|
# | 10 | 27m 49s | 0 | 34m 53s | 7m 4s (cold) | [2][] |
|
|
# | 20 | 24m 16s | 0 | 26m 46s | 2m 30s (hot) | [3][] |
|
|
# | 20 | 24m 15s | 1 | 30m 15s | 6m 0s (cold) | [4][] |
|
|
# | 30 | 23m 42s | 0 | 26m 35s | 2m 53s (hot) | [5][] |
|
|
# | 40 | 26m 26s | 26 | 29m 29s | 3m 3s (hot) | [6][] |
|
|
# | 50 | interrupted (*) | | | (hot) | [7][] |
|
|
#
|
|
# The startup time is calculated by substracting the "started time" visible
|
|
# on the Prow UI with the first line that has a timestamp. This time
|
|
# depends on whether this Kubernetes node already has a cache or not.
|
|
#
|
|
# These results have no statistical significance since each line is the
|
|
# result of a single Prow job. But these results still show that 10 is a
|
|
# good number.
|
|
#
|
|
# (*) It seems like at 50 nodes the pod gets killed somehow.
|
|
#
|
|
# [1]: https://prow.build-infra.jetstack.net/view/gs/jetstack-logs/pr-logs/pull/cert-manager_cert-manager/4968/pull-cert-manager-make-e2e-v1-23/1507028321639075840
|
|
# [2]: https://prow.build-infra.jetstack.net/view/gs/jetstack-logs/pr-logs/pull/cert-manager_cert-manager/4968/pull-cert-manager-make-e2e-v1-23/1507002589567258624
|
|
# [3]: https://prow.build-infra.jetstack.net/view/gs/jetstack-logs/pr-logs/pull/cert-manager_cert-manager/4968/pull-cert-manager-make-e2e-v1-23/1506994096810496000
|
|
# [4]: https://prow.build-infra.jetstack.net/view/gs/jetstack-logs/pr-logs/pull/cert-manager_cert-manager/4968/pull-cert-manager-make-e2e-v1-23/1506974361645486080
|
|
# [5]: https://prow.build-infra.jetstack.net/view/gs/jetstack-logs/pr-logs/pull/cert-manager_cert-manager/4968/pull-cert-manager-make-e2e-v1-23/1507011895024947200
|
|
# [6]: https://prow.build-infra.jetstack.net/view/gs/jetstack-logs/pr-logs/pull/cert-manager_cert-manager/4968/pull-cert-manager-make-e2e-v1-23/1507019887451574272
|
|
# [7]: https://prow.build-infra.jetstack.net/view/gs/jetstack-logs/pr-logs/pull/cert-manager_cert-manager/4968/pull-cert-manager-make-e2e-v1-23/1507040653668782080
|
|
|
|
nodes=40
|
|
|
|
flake_attempts=1
|
|
|
|
ginkgo_skip=
|
|
ginkgo_focus=
|
|
|
|
feature_gates=AdditionalCertificateOutputFormats=true,ExperimentalCertificateSigningRequestControllers=true,ExperimentalGatewayAPISupport=true,LiteralCertificateSubject=true,OtherNames=true
|
|
|
|
artifacts="./$BINDIR/artifacts"
|
|
|
|
help() {
|
|
cat <<EOF | color ""
|
|
Runs the end-to-end test suite against an already configured kind cluster.
|
|
|
|
Usage:
|
|
${bold}$(basename "$0") [--help] [args-for-ginkgo]${end}
|
|
|
|
Examples:
|
|
${bold}$(basename "$0") --ginkgo.skip='Venafi TPP|Venafi Cloud'${end}
|
|
${bold}$(basename "$0") --gingko.focus '.*a failing certificate that had a wrong dns name' --ginkgo.v --test.v -v=4${end}
|
|
|
|
Environment variables:
|
|
${green}GINKGO_FOCUS${end}
|
|
If set, only run the test suite that matches the given regex. It is
|
|
identical to running ${bold}--ginkgo.focus${end} on the command line.
|
|
This environment variable is useful when running this script from make.
|
|
For example:
|
|
${bold}make e2e GINKGO_SKIP='.*had a wrong dns name'${end}
|
|
${green}GINKGO_SKIP${end}
|
|
If set, skip the test suite that matches the given regex. It is
|
|
identical to running ${bold}--ginkgo.skip${end} on the command line.
|
|
${green}FLAKE_ATTEMPTS${end}
|
|
The number of times to attempt to run each test case before giving up.
|
|
The default is $flake_attempts.
|
|
${green}NODES${end}
|
|
Ginkgo's parallelism. The default is $nodes.
|
|
${green}FEATURE_GATES${end}
|
|
The feature gates that cert-manager is currently running with. Defaults
|
|
to $feature_gates
|
|
${green}ARTIFACTS${end}
|
|
The path to a directory where the JUnit XML files will be stored. By
|
|
default, the JUnit XML files are saved to $artifacts
|
|
|
|
Details:
|
|
Imagine you got the following failure:
|
|
|
|
${gray}1 |${end} ${red}• Failure [60.079 seconds]${end}
|
|
${gray}2 |${end} [Conformance] Certificates
|
|
${gray}3 |${end} ${gray}test/e2e/framework/framework.go:287${end}
|
|
${gray}4 |${end} with an External Issuer
|
|
${gray}5 |${end} ${gray}test/e2e/suite/conformance/certificates/tests.go:48${end}
|
|
${gray}6 |${end} Creating a Gateway [It]
|
|
${gray}7 |${end} ${gray}test/e2e/suite/conformance/certificates/suite.go:105${end}
|
|
|
|
You need to "reconstruct" the name of the test case. The ending [It] must be
|
|
removed. In the above example, the name of the test is:
|
|
|
|
[Conformance] Certificates with an External Issuer Creating a Gateway
|
|
${gray}<------------------------> <---------------------> <---------------->${end}
|
|
${gray} line 2 line 4 line 6${end}
|
|
|
|
To re-run this specific test case, you can use the following command:
|
|
|
|
${bold}$(basename "$0") --ginkgo.focus '\[Conformance\] Certificates with an External Issuer Creating a Gateway'${end}
|
|
|
|
If you want, you can match the end of the test case name:
|
|
|
|
${bold}$(basename "$0") --ginkgo.focus '.*Creating a Gateway'${end}
|
|
|
|
Note that if you use GINKGO_FOCUS or --ginkgo.focus, Ginkgo's parallelism will
|
|
be turned off in order to see the logs streamed (instead of waiting until test
|
|
ends before being able to see the logs).
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
if [ $# -gt 0 ]; then
|
|
case "$1" in
|
|
-h | --help)
|
|
help
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
for v in FEATURE_GATES FLAKE_ATTEMPTS NODES GINKGO_FOCUS GINKGO_SKIP ARTIFACTS; do
|
|
if printenv "$v" >/dev/null && [ -n "${!v}" ]; then
|
|
eval "$(tr '[:upper:]' '[:lower:]' <<<"$v")=\"${!v}\""
|
|
fi
|
|
done
|
|
|
|
ginkgo_args=("$@")
|
|
|
|
if [[ -n "$ginkgo_focus" ]]; then ginkgo_args+=(--ginkgo.focus="${ginkgo_focus}"); fi
|
|
if [[ -n "$ginkgo_skip" ]]; then ginkgo_args+=(--ginkgo.skip="${ginkgo_skip}"); fi
|
|
|
|
|
|
# Ginkgo doesn't stream the logs when running in parallel (--nodes). Let's
|
|
# disable parallelism to force Ginkgo to stream the logs when
|
|
# --ginkgo.focus or GINKGO_FOCUS is set, since --ginkgo.focus and
|
|
# GINKGO_FOCUS are often used to debug a specific test.
|
|
if [[ "${ginkgo_args[*]}" =~ ginkgo.focus ]]; then
|
|
nodes=1
|
|
ginkgo_args+=(--ginkgo.v --test.v)
|
|
fi
|
|
|
|
ginkgo_color=
|
|
|
|
if ! should_color; then
|
|
ginkgo_color="--no-color"
|
|
fi
|
|
|
|
mkdir -p "$artifacts"
|
|
|
|
export CGO_ENABLED=0
|
|
|
|
trace ginkgo \
|
|
--tags=e2e_test \
|
|
--procs="$nodes" \
|
|
--output-dir="$artifacts" \
|
|
--junit-report="junit__01.xml" \
|
|
--flake-attempts="$flake_attempts" \
|
|
--timeout="1h" \
|
|
$ginkgo_color \
|
|
-v \
|
|
--randomize-all \
|
|
--trace \
|
|
--poll-progress-after=60s \
|
|
./test/e2e/ \
|
|
-- \
|
|
--repo-root="$PWD" \
|
|
--report-dir="$artifacts" \
|
|
--acme-dns-server="${SERVICE_IP_PREFIX}.16" \
|
|
--acme-ingress-ip="${SERVICE_IP_PREFIX}.15" \
|
|
--acme-gateway-ip="${SERVICE_IP_PREFIX}.14" \
|
|
--ingress-controller-domain=ingress-nginx.http01.example.com \
|
|
--gateway-domain=gateway.http01.example.com \
|
|
--feature-gates="$feature_gates" \
|
|
"${ginkgo_args[@]}"
|