From 27e1a770b333369d0224773df23819548ab203ec Mon Sep 17 00:00:00 2001 From: Ashley Davis Date: Thu, 30 Jun 2022 18:18:30 +0100 Subject: [PATCH 1/3] add warning about vendor-go gotcha Signed-off-by: Ashley Davis --- make/tools.mk | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/make/tools.mk b/make/tools.mk index 07c851a7a..4c18ba33c 100644 --- a/make/tools.mk +++ b/make/tools.mk @@ -436,6 +436,12 @@ else # Although we "vendor" most tools in $(BINDIR)/tools, we still require some binaries # to be available on the system. The vendor-go MAKECMDGOALS trick prevents the # check for the presence of Go when 'make vendor-go' is run. + +# Gotcha warning: MAKECMDGOALS only contains what the _top level_ make invocation used, and doesn't look at target dependencies +# i.e. if we have a target "abc: vendor-go test" and run "make abc", we'll get an error +# about go being missing even though abc itself depends on vendor-go! +# That means we need to pass vendor-go at the top level if go is not installed (i.e. "make vendor-go abc") + MISSING=$(shell (command -v curl >/dev/null || echo curl) \ && (command -v jq >/dev/null || echo jq) \ && (command -v sha256sum >/dev/null || echo sha256sum) \ From ca356962440f409f603529dc4cf35cfd22dcf158 Mon Sep 17 00:00:00 2001 From: Ashley Davis Date: Thu, 30 Jun 2022 16:51:00 +0100 Subject: [PATCH 2/3] add make-based upgrade test This uses cmctl instead of kubectl_cert-manager, uses make instead of bazel and fixes an incorrect container name in test/fixtures/upgrade/overlay/cainjector-ops.yaml Signed-off-by: Ashley Davis --- hack/verify-upgrade.sh | 177 +++++++++--------- make/ci.mk | 5 +- make/git.mk | 4 + make/test.mk | 4 + .../upgrade/overlay/cainjector-ops.yaml | 2 +- .../upgrade/overlay/controller-ops.yaml | 2 +- .../fixtures/upgrade/overlay/webhook-ops.yaml | 2 +- 7 files changed, 98 insertions(+), 98 deletions(-) diff --git a/hack/verify-upgrade.sh b/hack/verify-upgrade.sh index 2441d8402..be7b6ac27 100755 --- a/hack/verify-upgrade.sh +++ b/hack/verify-upgrade.sh @@ -20,61 +20,59 @@ set -o pipefail SCRIPT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" export REPO_ROOT="${SCRIPT_ROOT}/.." -source "${REPO_ROOT}/devel/lib/lib.sh" source "${REPO_ROOT}/hack/build/version.sh" -export APP_VERSION="$(date +"%s")" kube::version::last_published_release LATEST_RELEASE="${KUBE_LAST_RELEASE}" -# Ensure helm, kind, kubectl, ytt are available -echo "Building the required tools.." -bazel build //hack/bin:helm //hack/bin:kind //hack/bin:ytt //hack/bin:kubectl //hack/bin:kubectl-cert_manager -bindir="$(bazel info bazel-bin)" -export PATH="${bindir}/hack/bin/:$PATH" +usage_and_exit() { + echo "usage: $0 " >&2 + exit 1 +} -# Build images from latest master and load into the kind cluster. These will be -# used when upgrading with both kubectl and helm. -# Tag images with APP_VERSION for consistency with devel/addon/certmanager/install.sh. -echo "Building latest cert-manger images.." -APP_VERSION=${APP_VERSION} \ -bazel run \ - --stamp=true \ - --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \ - "//devel/addon/certmanager:bundle" +if [[ -z "${1:-}" || -z "${2:-}" || -z "${3:-}" ||-z "${4:-}" || -z "${5:-}" ]]; then + usage_and_exit +fi -echo "Loading latest cert-manager images to cluster.." -load_image "quay.io/jetstack/cert-manager-controller:${APP_VERSION}" & -load_image "quay.io/jetstack/cert-manager-acmesolver:${APP_VERSION}" & -load_image "quay.io/jetstack/cert-manager-cainjector:${APP_VERSION}" & -load_image "quay.io/jetstack/cert-manager-webhook:${APP_VERSION}" & -load_image "quay.io/jetstack/cert-manager-ctl:${APP_VERSION}" & -wait +helm=$(realpath "$1") +kind=$(realpath "$2") +ytt=$(realpath "$3") +kubectl=$(realpath "$4") +cmctl=$(realpath "$5") -# Namespace to deploy into +# Set up a fresh kind cluster + +$kind delete clusters kind || : +make e2e-setup-kind + +################################################ +# VERIFY INSTALL, UPGRADE, UNINSTALL WITH HELM # +################################################ + +# Namespace we'll deploy into NAMESPACE="${NAMESPACE:-cert-manager}" + # Release name to use with Helm RELEASE_NAME="${RELEASE_NAME:-cert-manager}" -# cert-manager Helm chart -HELM_CHART="jetstack/cert-manager" -############ -# VERIFY INSTALL, UPGRADE, UNINSTALL WITH HELM -############ +HELM_URL="https://charts.jetstack.io" -echo "Testing upgrade from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} with Helm.." +# cert-manager Helm chart location +HELM_CHART="cmupgradetest/cert-manager" + +echo "+++ Testing upgrading from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} with Helm" # This will target the host's helm repository cache -helm repo add jetstack https://charts.jetstack.io -helm repo update +$helm repo add cmupgradetest $HELM_URL +$helm repo update # 1. INSTALL THE LATEST PUBLISHED HELM CHART -echo "Installing cert-manager ${LATEST_RELEASE} Helm chart into the cluster..." +echo "+++ Installing cert-manager ${LATEST_RELEASE} Helm chart into the cluster..." # Upgrade or install latest published cert-manager Helm release -helm upgrade \ +$helm upgrade \ --install \ --wait \ --namespace "${NAMESPACE}" \ @@ -85,102 +83,98 @@ helm upgrade \ "$HELM_CHART" # Wait for the cert-manager api to be available -kubectl cert-manager check api --wait=2m -v +$cmctl check api --wait=2m -v -echo "Creating some cert-manager resources.." +echo "+++ Creating some cert-manager resources.." # Create a cert-manager issuer and cert -kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first" +$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first" # Ensure cert becomes ready -kubectl wait --for=condition=Ready cert/test1 --timeout=180s +$kubectl wait --for=condition=Ready cert/test1 --timeout=180s # 2. BUILD AND UPGRADE TO HELM CHART FROM THE CURRENT MASTER -bazel build //deploy/charts/cert-manager - -echo "Upgrading cert-manager Helm release to commit ${KUBE_GIT_COMMIT}..." -helm upgrade \ - --install \ - --wait \ - --namespace "${NAMESPACE}" \ - --set installCRDs=true \ - --create-namespace \ - "$RELEASE_NAME" \ - "$REPO_ROOT/bazel-bin/deploy/charts/cert-manager/cert-manager.tgz" +# e2e-setup-certamanager both builds and deploys the latest available chart based on the current checkout +make e2e-setup-certmanager # Wait for the cert-manager api to be available -kubectl cert-manager check api --wait=2m -v +$cmctl check api --wait=2m -v # Test that the existing cert-manager resources can still be retrieved -kubectl get issuer/selfsigned-issuer cert/test1 +$kubectl get issuer/selfsigned-issuer cert/test1 -echo "Creating some cert-manager resources.." +echo "+++ Creating some more cert-manager resources.." -# # Create another certificate -kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second" +# Create another certificate +$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second" # Ensure cert becomes ready -kubectl wait --for=condition=Ready cert/test2 --timeout=180s +$kubectl wait --for=condition=Ready cert/test2 --timeout=180s # 3. UNINSTALL HELM RELEASE -echo "Uninstalling the Helm release.." -kubectl delete \ - -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" +echo "+++ Uninstalling the Helm release" -helm uninstall \ +$kubectl delete -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" + +$helm uninstall \ --namespace "${NAMESPACE}" \ "$RELEASE_NAME" -kubectl delete "namespace/${NAMESPACE}" \ - --wait +$kubectl delete "namespace/${NAMESPACE}" --wait -############ -# VERIFY INSTALL, UPGRADE, UNINSTALL WITH STATIC MANIFESTS -############ +############################################################ +# VERIFY INSTALL, UPGRADE, UNINSTALL WITH STATIC MANIFESTS # +############################################################ # 1. INSTALL THE LATEST PUBLISHED RELEASE WITH STATIC MANIFESTS -echo "Testing cert-manager upgrade from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} with static manifests.." +echo "+++ Testing cert-manager upgrade from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} using static manifests" -echo "Install cert-manager ${LATEST_RELEASE} using static manifests.." -kubectl apply \ +echo "+++ Installing cert-manager ${LATEST_RELEASE} using static manifests" + +$kubectl apply \ -f "https://github.com/cert-manager/cert-manager/releases/download/${LATEST_RELEASE}/cert-manager.yaml" \ --wait -kubectl wait \ +$kubectl wait \ --for=condition=available \ --timeout=180s deployment/cert-manager-webhook \ --namespace "${NAMESPACE}" # Wait for the cert-manager api to be available -kubectl cert-manager check api --wait=2m -v +$cmctl check api --wait=2m -v # Create a cert-manager issuer and cert -kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first" +$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first" # Ensure cert becomes ready -kubectl wait --for=condition=Ready cert/test1 --timeout=180s +$kubectl wait --for=condition=Ready cert/test1 --timeout=180s # 2. VERIFY UPGRADE TO THE LATEST BUILD FROM MASTER -echo "Install cert-manager commit ${KUBE_GIT_COMMIT} using static manifests.." +MANIFEST_LOCATION=${REPO_ROOT}/_bin/yaml/cert-manager.yaml + +echo "+++ Installing cert-manager commit ${KUBE_GIT_COMMIT} using static manifests" # Build the static manifests -bazel build //deploy/manifests +make release-manifests + +RELEASE_VERSION=$(make --silent release-version) # Overwrite image tags in the static manifests and deploy. -ytt -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/controller-ops.yaml" \ - -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/cainjector-ops.yaml" \ - -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/webhook-ops.yaml" \ - -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/values.yaml" \ - -f "${REPO_ROOT}/bazel-bin/deploy/manifests/cert-manager.yaml" \ - --data-value app_version="${APP_VERSION}" \ - --ignore-unknown-comments | kubectl apply -f - +$ytt -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/controller-ops.yaml" \ + -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/cainjector-ops.yaml" \ + -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/webhook-ops.yaml" \ + -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/values.yaml" \ + -f $MANIFEST_LOCATION \ + --data-value app_version="${RELEASE_VERSION}" \ + --ignore-unknown-comments | kubectl apply -f - -rollout_cmd="kubectl rollout status deployment/cert-manager-webhook --namespace ${NAMESPACE}" +rollout_cmd="$kubectl rollout status deployment/cert-manager-webhook --namespace ${NAMESPACE}" attempts=0 + until $rollout_cmd; do $rollout_cmd ((attempts++)) @@ -192,22 +186,21 @@ until $rollout_cmd; do done # Wait for the cert-manager api to be available -kubectl cert-manager check api --wait=2m -v +$cmctl check api --wait=2m -v # Test that the existing cert-manager resources can still be retrieved -kubectl get issuer/selfsigned-issuer cert/test1 +$kubectl get issuer/selfsigned-issuer cert/test1 -echo "Creating some cert-manager resources.." +echo "+++ Creating some cert-manager resources" -# # Create another certificate -kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second" +# Create another certificate +$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second" # Ensure cert becomes ready -kubectl wait --for=condition=Ready cert/test2 --timeout=180s - -echo "Uninstalling cert-manager.." +$kubectl wait --for=condition=Ready cert/test2 --timeout=180s # 3. UNINSTALL -kubectl delete \ - -f "${REPO_ROOT}/bazel-bin/deploy/manifests/cert-manager.yaml" \ - --wait \ + +echo "+++ Uninstalling cert-manager" + +$kubectl delete -f $MANIFEST_LOCATION --wait diff --git a/make/ci.mk b/make/ci.mk index 0a5ece020..e9aeb4200 100644 --- a/make/ci.mk +++ b/make/ci.mk @@ -112,9 +112,8 @@ verify_chart: ./hack/verify-chart-version.sh bazel-bin/deploy/charts/cert-manager/cert-manager.tgz .PHONY: verify_upgrade -verify_upgrade: - $(warning "The 'verify_upgrade' target is deprecated and will be removed soon. Please use instead 'make e2e-setup-kind && ./hack/verify-upgrade.sh'.") - ./hack/verify-upgrade.sh +verify_upgrade: test-upgrade + $(warning "The 'verify_upgrade' target is deprecated and will be removed soon. Please use instead 'make test-upgrade'.") .PHONY: cluster cluster: diff --git a/make/git.mk b/make/git.mk index d3dee6920..50f2803c9 100644 --- a/make/git.mk +++ b/make/git.mk @@ -13,6 +13,10 @@ gitver: @echo "Is prerelease: \"$(IS_PRERELEASE)\"" @echo "Git commit hash: \"$(GITCOMMIT)\"" +.PHONY: release-version +release-version: + @echo "$(RELEASE_VERSION)" + # Lists all remote tags on the upstream, which gives tags in format: # " ref/tags/". Strips commit + tag prefix, filters out tags for v1+, # and manually removes v1.2.0-alpha.1, since that version's manifest contains diff --git a/make/test.mk b/make/test.mk index c775bf1b4..46e77f4eb 100644 --- a/make/test.mk +++ b/make/test.mk @@ -75,6 +75,10 @@ e2e: $(BINDIR)/scratch/kind-exists $(BINDIR)/tools/kubectl $(BINDIR)/tools/ginkg e2e-ci: e2e-setup-kind e2e-setup $(MAKE) --no-print-directory e2e FLAKE_ATTEMPTS=2 K8S_VERSION="$(K8S_VERSION)" || ($(MAKE) kind-logs && exit 1) +.PHONY: test-upgrade +test-upgrade: | $(BINDIR)/tools/helm $(BINDIR)/tools/kind $(BINDIR)/tools/ytt $(BINDIR)/tools/kubectl $(BINDIR)/cmctl/cmctl-$(HOST_OS)-$(HOST_ARCH) + ./hack/verify-upgrade.sh $(BINDIR)/tools/helm $(BINDIR)/tools/kind $(BINDIR)/tools/ytt $(BINDIR)/tools/kubectl $(BINDIR)/cmctl/cmctl-$(HOST_OS)-$(HOST_ARCH) + test/integration/versionchecker/testdata/test_manifests.tar: $(BINDIR)/scratch/oldcrds.tar $(BINDIR)/yaml/cert-manager.yaml @# Remove the temp files if they exist rm -f $(BINDIR)/scratch/versionchecker-test-manifests.tar $(BINDIR)/scratch/$(RELEASE_VERSION).yaml diff --git a/test/fixtures/upgrade/overlay/cainjector-ops.yaml b/test/fixtures/upgrade/overlay/cainjector-ops.yaml index 5267ef2ef..890800eff 100644 --- a/test/fixtures/upgrade/overlay/cainjector-ops.yaml +++ b/test/fixtures/upgrade/overlay/cainjector-ops.yaml @@ -22,4 +22,4 @@ spec: spec: containers: #@overlay/match by=overlay.subset({"name": "cert-manager"}) - - image: #@ "quay.io/jetstack/cert-manager-controller:{}".format(data.values.app_version) + - image: #@ "docker.io/library/cert-manager-cainjector-amd64:{}".format(data.values.app_version) diff --git a/test/fixtures/upgrade/overlay/controller-ops.yaml b/test/fixtures/upgrade/overlay/controller-ops.yaml index 20dc73c9b..6a86debae 100644 --- a/test/fixtures/upgrade/overlay/controller-ops.yaml +++ b/test/fixtures/upgrade/overlay/controller-ops.yaml @@ -22,5 +22,5 @@ spec: spec: containers: #@overlay/match by=overlay.subset({"name": "cert-manager"}) - - image: #@ "quay.io/jetstack/cert-manager-controller:{}".format(data.values.app_version) + - image: #@ "docker.io/library/cert-manager-controller-amd64:{}".format(data.values.app_version) diff --git a/test/fixtures/upgrade/overlay/webhook-ops.yaml b/test/fixtures/upgrade/overlay/webhook-ops.yaml index 31376706b..dea974404 100644 --- a/test/fixtures/upgrade/overlay/webhook-ops.yaml +++ b/test/fixtures/upgrade/overlay/webhook-ops.yaml @@ -22,4 +22,4 @@ spec: spec: containers: #@overlay/match by=overlay.subset({"name": "cert-manager"}) - - image: #@ "quay.io/jetstack/cert-manager-webhook:{}".format(data.values.app_version) + - image: #@ "docker.io/library/cert-manager-webhook-amd64:{}".format(data.values.app_version) From 18e98ce3da782d7c13ce905119d60e03c65804b2 Mon Sep 17 00:00:00 2001 From: Ashley Davis Date: Thu, 30 Jun 2022 18:15:01 +0100 Subject: [PATCH 3/3] move legacy targets to legacy.mk we don't _need_ to remove these and we can keep them around for longer, but we don't need them to be in files we actually use and edit. putting the targets in a separate file feels cleaner! Signed-off-by: Ashley Davis --- Makefile | 1 + make/ci.mk | 37 ------------------------------------- make/legacy.mk | 23 +++++++++++++++++++++++ make/tools.mk | 18 ------------------ 4 files changed, 24 insertions(+), 55 deletions(-) create mode 100644 make/legacy.mk diff --git a/Makefile b/Makefile index 6ebc8dd77..b622be30c 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,7 @@ include make/release.mk include make/manifests.mk include make/licenses.mk include make/e2e-setup.mk +include make/legacy.mk include make/help.mk .PHONY: clean diff --git a/make/ci.mk b/make/ci.mk index e9aeb4200..845ed6a2d 100644 --- a/make/ci.mk +++ b/make/ci.mk @@ -82,40 +82,3 @@ update-codegen: | k8s-codegen-tools $(DEPENDS_ON_GO) ## ## @category Development update-all: update-crds update-codegen update-licenses - -# The targets (verify_deps, verify_chart, verify_upgrade, and cluster) are -# temorary and exist to keep the compatibility with the following Prow jobs: -# -# pull-cert-manager-chart -# pull-cert-manager-deps -# pull-cert-manager-upgrade -# -# These targets should be removed as soon as the four above jobs and scripts are -# updated to use the "make" flow. -.PHONY: verify -verify: - $(warning "The 'verify' target is deprecated and will be removed soon. Please use instead 'ci-presubmit'") - bazel test //... - -.PHONY: verify_deps -verify_deps: - @# this target can be removed once we've removed the pull-cert-manager-deps test from presubmits - @# for now, just make it a no-op so the tests don't fail - $(warning "The 'verify_deps' target is deprecated, does nothing, and will be removed soon. This target is not useful anymore with the new make flow.") - @true - -# requires docker -.PHONY: verify_chart -verify_chart: - $(warning "The 'verify_chart' target is deprecated and will be removed soon. Please use instead 'verify-chart'.") - bazel build //deploy/charts/cert-manager - ./hack/verify-chart-version.sh bazel-bin/deploy/charts/cert-manager/cert-manager.tgz - -.PHONY: verify_upgrade -verify_upgrade: test-upgrade - $(warning "The 'verify_upgrade' target is deprecated and will be removed soon. Please use instead 'make test-upgrade'.") - -.PHONY: cluster -cluster: - $(warning "The 'cluster' target is deprecated and will be removed soon. Please use instead 'make e2e-setup-kind'.") - ./devel/ci-cluster.sh diff --git a/make/legacy.mk b/make/legacy.mk new file mode 100644 index 000000000..bdaddef27 --- /dev/null +++ b/make/legacy.mk @@ -0,0 +1,23 @@ +# Targets in this file are legacy holdovers from before the migration to make. +# They're preserved here in case they're used in some third party CI system or script, +# but are liable to being removed or broken without warning. + +.PHONY: verify +verify: ci-presubmit test + $(warning "The '$@' target is deprecated and may be removed. Use 'make $^' instead.") + +.PHONY: verify_deps +verify_deps: + $(warning "The '$@' target is deprecated and may be removed. This target is a no-op with the new make flow.") + +.PHONY: cluster +cluster: e2e-setup-kind + $(warning "The '$@' target is deprecated and may be removed. Use 'make $^' instead.") + +.PHONY: verify_chart +verify_chart: verify-chart + $(warning "The '$@' target is deprecated and may be removed. Use 'make $^' instead.") + +.PHONY: verify_upgrade +verify_upgrade: test-upgrade + $(warning "The '$@' target is deprecated and may be removed. Use 'make $^' instead.") diff --git a/make/tools.mk b/make/tools.mk index 4c18ba33c..e83f1aead 100644 --- a/make/tools.mk +++ b/make/tools.mk @@ -416,23 +416,6 @@ $(BINDIR)/downloaded/gatewayapi-v%: | $(BINDIR)/downloaded $(BINDIR)/tools $(BINDIR)/downloaded $(BINDIR)/downloaded/tools: @mkdir -p $@ -# The targets (verify_deps, verify_chart, verify_upgrade, and cluster) are -# temorary and exist to keep the compatibility with the following Prow jobs: -# -# pull-cert-manager-chart -# pull-cert-manager-deps -# pull-cert-manager-upgrade -# -# Until we have removed these Bazel-based targets, we must disable the check -# of the system tools since the Bazel targets don't rely on those, and the image -# -# eu.gcr.io/jetstack-build-infra-images/bazelbuild -# -# doesn't have these tools. -BAZEL_TARGET := $(filter verify verify_deps verify_chart verify_upgrade cluster,$(MAKECMDGOALS)) -ifneq ($(BAZEL_TARGET),) -$(warning Not checking whether the system tools are present since Bazel already takes care of that in the target $(MAKECMDGOALS). .) -else # Although we "vendor" most tools in $(BINDIR)/tools, we still require some binaries # to be available on the system. The vendor-go MAKECMDGOALS trick prevents the # check for the presence of Go when 'make vendor-go' is run. @@ -452,7 +435,6 @@ MISSING=$(shell (command -v curl >/dev/null || echo curl) \ ifneq ($(MISSING),) $(error Missing required tools: $(MISSING)) endif -endif .PHONY: update-kind-images update-kind-images: $(BINDIR)/tools/crane