From ca356962440f409f603529dc4cf35cfd22dcf158 Mon Sep 17 00:00:00 2001
From: Ashley Davis <ashley.davis@jetstack.io>
Date: Thu, 30 Jun 2022 16:51:00 +0100
Subject: [PATCH] add make-based upgrade test

This uses cmctl instead of kubectl_cert-manager, uses make instead of
bazel and fixes an incorrect container name in
test/fixtures/upgrade/overlay/cainjector-ops.yaml

Signed-off-by: Ashley Davis <ashley.davis@jetstack.io>
---
 hack/verify-upgrade.sh                        | 177 +++++++++---------
 make/ci.mk                                    |   5 +-
 make/git.mk                                   |   4 +
 make/test.mk                                  |   4 +
 .../upgrade/overlay/cainjector-ops.yaml       |   2 +-
 .../upgrade/overlay/controller-ops.yaml       |   2 +-
 .../fixtures/upgrade/overlay/webhook-ops.yaml |   2 +-
 7 files changed, 98 insertions(+), 98 deletions(-)

diff --git a/hack/verify-upgrade.sh b/hack/verify-upgrade.sh
index 2441d8402..be7b6ac27 100755
--- a/hack/verify-upgrade.sh
+++ b/hack/verify-upgrade.sh
@@ -20,61 +20,59 @@ set -o pipefail
 
 SCRIPT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )"
 export REPO_ROOT="${SCRIPT_ROOT}/.."
-source "${REPO_ROOT}/devel/lib/lib.sh"
 source "${REPO_ROOT}/hack/build/version.sh"
-export APP_VERSION="$(date +"%s")"
 
 kube::version::last_published_release
 
 LATEST_RELEASE="${KUBE_LAST_RELEASE}"
 
-# Ensure helm, kind, kubectl, ytt are available
-echo "Building the required tools.."
-bazel build //hack/bin:helm //hack/bin:kind //hack/bin:ytt //hack/bin:kubectl //hack/bin:kubectl-cert_manager
-bindir="$(bazel info bazel-bin)"
-export PATH="${bindir}/hack/bin/:$PATH"
+usage_and_exit() {
+	echo "usage: $0 <path-to-helm> <path-to-kind> <path-to-ytt> <path-to-kubectl> <path-to-cmctl>" >&2
+	exit 1
+}
 
-# Build images from latest master and load into the kind cluster. These will be
-# used when upgrading with both kubectl and helm.
-# Tag images with APP_VERSION for consistency with devel/addon/certmanager/install.sh.
-echo "Building latest cert-manger images.."
-APP_VERSION=${APP_VERSION} \
-bazel run \
-  --stamp=true \
-  --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \
-  "//devel/addon/certmanager:bundle"
+if [[ -z "${1:-}" || -z "${2:-}" || -z "${3:-}" ||-z "${4:-}" || -z "${5:-}" ]]; then
+	usage_and_exit
+fi
 
-echo "Loading latest cert-manager images to cluster.."
-load_image "quay.io/jetstack/cert-manager-controller:${APP_VERSION}" &
-load_image "quay.io/jetstack/cert-manager-acmesolver:${APP_VERSION}" &
-load_image "quay.io/jetstack/cert-manager-cainjector:${APP_VERSION}" &
-load_image "quay.io/jetstack/cert-manager-webhook:${APP_VERSION}" &
-load_image "quay.io/jetstack/cert-manager-ctl:${APP_VERSION}" &
-wait
+helm=$(realpath "$1")
+kind=$(realpath "$2")
+ytt=$(realpath "$3")
+kubectl=$(realpath "$4")
+cmctl=$(realpath "$5")
 
-# Namespace to deploy into
+# Set up a fresh kind cluster
+
+$kind delete clusters kind || :
+make e2e-setup-kind
+
+################################################
+# VERIFY INSTALL, UPGRADE, UNINSTALL WITH HELM #
+################################################
+
+# Namespace we'll deploy into
 NAMESPACE="${NAMESPACE:-cert-manager}"
+
 # Release name to use with Helm
 RELEASE_NAME="${RELEASE_NAME:-cert-manager}"
-# cert-manager Helm chart
-HELM_CHART="jetstack/cert-manager"
 
-############
-# VERIFY INSTALL, UPGRADE, UNINSTALL WITH HELM
-############
+HELM_URL="https://charts.jetstack.io"
 
-echo "Testing upgrade from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} with Helm.."
+# cert-manager Helm chart location
+HELM_CHART="cmupgradetest/cert-manager"
+
+echo "+++ Testing upgrading from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} with Helm"
 
 # This will target the host's helm repository cache
-helm repo add jetstack https://charts.jetstack.io
-helm repo update
+$helm repo add cmupgradetest $HELM_URL
+$helm repo update
 
 # 1. INSTALL THE LATEST PUBLISHED HELM CHART
 
-echo "Installing cert-manager ${LATEST_RELEASE} Helm chart into the cluster..."
+echo "+++ Installing cert-manager ${LATEST_RELEASE} Helm chart into the cluster..."
 
 # Upgrade or install latest published cert-manager Helm release
-helm upgrade \
+$helm upgrade \
     --install \
     --wait \
     --namespace "${NAMESPACE}" \
@@ -85,102 +83,98 @@ helm upgrade \
     "$HELM_CHART"
 
 # Wait for the cert-manager api to be available
-kubectl cert-manager check api --wait=2m -v
+$cmctl check api --wait=2m -v
 
-echo "Creating some cert-manager resources.."
+echo "+++ Creating some cert-manager resources.."
 
 # Create a cert-manager issuer and cert
-kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first"
+$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first"
 
 # Ensure cert becomes ready
-kubectl wait --for=condition=Ready cert/test1 --timeout=180s
+$kubectl wait --for=condition=Ready cert/test1 --timeout=180s
 
 # 2. BUILD AND UPGRADE TO HELM CHART FROM THE CURRENT MASTER
 
-bazel build //deploy/charts/cert-manager
-
-echo "Upgrading cert-manager Helm release to commit ${KUBE_GIT_COMMIT}..."
-helm upgrade \
-    --install \
-    --wait \
-    --namespace "${NAMESPACE}" \
-    --set installCRDs=true \
-    --create-namespace \
-    "$RELEASE_NAME" \
-    "$REPO_ROOT/bazel-bin/deploy/charts/cert-manager/cert-manager.tgz"
+# e2e-setup-certamanager both builds and deploys the latest available chart based on the current checkout
+make e2e-setup-certmanager
 
 # Wait for the cert-manager api to be available
-kubectl cert-manager check api --wait=2m -v
+$cmctl check api --wait=2m -v
 
 # Test that the existing cert-manager resources can still be retrieved
-kubectl get issuer/selfsigned-issuer cert/test1
+$kubectl get issuer/selfsigned-issuer cert/test1
 
-echo "Creating some cert-manager resources.."
+echo "+++ Creating some more cert-manager resources.."
 
-# # Create another certificate
-kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second"
+# Create another certificate
+$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second"
 
 # Ensure cert becomes ready
-kubectl wait --for=condition=Ready cert/test2 --timeout=180s
+$kubectl wait --for=condition=Ready cert/test2 --timeout=180s
 
 # 3. UNINSTALL HELM RELEASE
-echo "Uninstalling the Helm release.."
 
-kubectl delete \
-	-f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml"
+echo "+++ Uninstalling the Helm release"
 
-helm uninstall \
+$kubectl delete -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml"
+
+$helm uninstall \
     --namespace "${NAMESPACE}" \
     "$RELEASE_NAME"
 
-kubectl delete "namespace/${NAMESPACE}" \
-	--wait
+$kubectl delete "namespace/${NAMESPACE}" --wait
 
-############
-# VERIFY INSTALL, UPGRADE, UNINSTALL WITH STATIC MANIFESTS
-############
+############################################################
+# VERIFY INSTALL, UPGRADE, UNINSTALL WITH STATIC MANIFESTS #
+############################################################
 
 # 1. INSTALL THE LATEST PUBLISHED RELEASE WITH STATIC MANIFESTS
 
-echo "Testing cert-manager upgrade from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} with static manifests.."
+echo "+++ Testing cert-manager upgrade from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} using static manifests"
 
-echo "Install cert-manager ${LATEST_RELEASE} using static manifests.."
-kubectl apply \
+echo "+++ Installing cert-manager ${LATEST_RELEASE} using static manifests"
+
+$kubectl apply \
 	-f "https://github.com/cert-manager/cert-manager/releases/download/${LATEST_RELEASE}/cert-manager.yaml" \
 	--wait
 
-kubectl wait \
+$kubectl wait \
 	--for=condition=available \
 	--timeout=180s deployment/cert-manager-webhook \
 	--namespace "${NAMESPACE}"
 
 # Wait for the cert-manager api to be available
-kubectl cert-manager check api --wait=2m -v
+$cmctl check api --wait=2m -v
 
 # Create a cert-manager issuer and cert
-kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first"
+$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first"
 
 # Ensure cert becomes ready
-kubectl wait --for=condition=Ready cert/test1 --timeout=180s
+$kubectl wait --for=condition=Ready cert/test1 --timeout=180s
 
 # 2. VERIFY UPGRADE TO THE LATEST BUILD FROM MASTER
 
-echo "Install cert-manager commit ${KUBE_GIT_COMMIT} using static manifests.."
+MANIFEST_LOCATION=${REPO_ROOT}/_bin/yaml/cert-manager.yaml
+
+echo "+++ Installing cert-manager commit ${KUBE_GIT_COMMIT} using static manifests"
 
 # Build the static manifests
-bazel build //deploy/manifests
+make release-manifests
+
+RELEASE_VERSION=$(make --silent release-version)
 
 # Overwrite image tags in the static manifests and deploy.
-ytt -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/controller-ops.yaml" \
-	-f "${REPO_ROOT}/test/fixtures/upgrade/overlay/cainjector-ops.yaml" \
-	-f "${REPO_ROOT}/test/fixtures/upgrade/overlay/webhook-ops.yaml" \
-	-f "${REPO_ROOT}/test/fixtures/upgrade/overlay/values.yaml" \
-	-f "${REPO_ROOT}/bazel-bin/deploy/manifests/cert-manager.yaml" \
-	--data-value app_version="${APP_VERSION}" \
-	--ignore-unknown-comments | kubectl apply -f -
+$ytt -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/controller-ops.yaml" \
+     -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/cainjector-ops.yaml" \
+     -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/webhook-ops.yaml" \
+     -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/values.yaml" \
+     -f $MANIFEST_LOCATION \
+     --data-value app_version="${RELEASE_VERSION}" \
+     --ignore-unknown-comments | kubectl apply -f -
 
-rollout_cmd="kubectl rollout status deployment/cert-manager-webhook --namespace ${NAMESPACE}"
+rollout_cmd="$kubectl rollout status deployment/cert-manager-webhook --namespace ${NAMESPACE}"
 attempts=0
+
 until $rollout_cmd; do
   $rollout_cmd
   ((attempts++))
@@ -192,22 +186,21 @@ until $rollout_cmd; do
 done
 
 # Wait for the cert-manager api to be available
-kubectl cert-manager check api --wait=2m -v
+$cmctl check api --wait=2m -v
 
 # Test that the existing cert-manager resources can still be retrieved
-kubectl get issuer/selfsigned-issuer cert/test1
+$kubectl get issuer/selfsigned-issuer cert/test1
 
-echo "Creating some cert-manager resources.."
+echo "+++ Creating some cert-manager resources"
 
-# # Create another certificate
-kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second"
+# Create another certificate
+$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second"
 
 # Ensure cert becomes ready
-kubectl wait --for=condition=Ready cert/test2 --timeout=180s
-
-echo "Uninstalling cert-manager.."
+$kubectl wait --for=condition=Ready cert/test2 --timeout=180s
 
 # 3. UNINSTALL
-kubectl delete \
-	-f "${REPO_ROOT}/bazel-bin/deploy/manifests/cert-manager.yaml" \
-	--wait \
+
+echo "+++ Uninstalling cert-manager"
+
+$kubectl delete -f $MANIFEST_LOCATION --wait
diff --git a/make/ci.mk b/make/ci.mk
index 0a5ece020..e9aeb4200 100644
--- a/make/ci.mk
+++ b/make/ci.mk
@@ -112,9 +112,8 @@ verify_chart:
 	./hack/verify-chart-version.sh bazel-bin/deploy/charts/cert-manager/cert-manager.tgz
 
 .PHONY: verify_upgrade
-verify_upgrade:
-	$(warning "The 'verify_upgrade' target is deprecated and will be removed soon. Please use instead 'make e2e-setup-kind && ./hack/verify-upgrade.sh'.")
-	./hack/verify-upgrade.sh
+verify_upgrade: test-upgrade
+	$(warning "The 'verify_upgrade' target is deprecated and will be removed soon. Please use instead 'make test-upgrade'.")
 
 .PHONY: cluster
 cluster:
diff --git a/make/git.mk b/make/git.mk
index d3dee6920..50f2803c9 100644
--- a/make/git.mk
+++ b/make/git.mk
@@ -13,6 +13,10 @@ gitver:
 	@echo "Is prerelease:     \"$(IS_PRERELEASE)\""
 	@echo "Git commit hash:   \"$(GITCOMMIT)\""
 
+.PHONY: release-version
+release-version:
+	@echo "$(RELEASE_VERSION)"
+
 # Lists all remote tags on the upstream, which gives tags in format:
 # "<commit> ref/tags/<tag>". Strips commit + tag prefix, filters out tags for v1+,
 # and manually removes v1.2.0-alpha.1, since that version's manifest contains
diff --git a/make/test.mk b/make/test.mk
index c775bf1b4..46e77f4eb 100644
--- a/make/test.mk
+++ b/make/test.mk
@@ -75,6 +75,10 @@ e2e: $(BINDIR)/scratch/kind-exists $(BINDIR)/tools/kubectl $(BINDIR)/tools/ginkg
 e2e-ci: e2e-setup-kind e2e-setup
 	$(MAKE) --no-print-directory e2e FLAKE_ATTEMPTS=2 K8S_VERSION="$(K8S_VERSION)" || ($(MAKE) kind-logs && exit 1)
 
+.PHONY: test-upgrade
+test-upgrade: | $(BINDIR)/tools/helm $(BINDIR)/tools/kind $(BINDIR)/tools/ytt $(BINDIR)/tools/kubectl $(BINDIR)/cmctl/cmctl-$(HOST_OS)-$(HOST_ARCH)
+	./hack/verify-upgrade.sh $(BINDIR)/tools/helm $(BINDIR)/tools/kind $(BINDIR)/tools/ytt $(BINDIR)/tools/kubectl $(BINDIR)/cmctl/cmctl-$(HOST_OS)-$(HOST_ARCH)
+
 test/integration/versionchecker/testdata/test_manifests.tar: $(BINDIR)/scratch/oldcrds.tar $(BINDIR)/yaml/cert-manager.yaml
 	@# Remove the temp files if they exist
 	rm -f $(BINDIR)/scratch/versionchecker-test-manifests.tar $(BINDIR)/scratch/$(RELEASE_VERSION).yaml
diff --git a/test/fixtures/upgrade/overlay/cainjector-ops.yaml b/test/fixtures/upgrade/overlay/cainjector-ops.yaml
index 5267ef2ef..890800eff 100644
--- a/test/fixtures/upgrade/overlay/cainjector-ops.yaml
+++ b/test/fixtures/upgrade/overlay/cainjector-ops.yaml
@@ -22,4 +22,4 @@ spec:
     spec:
       containers:
       #@overlay/match by=overlay.subset({"name": "cert-manager"})
-      - image: #@ "quay.io/jetstack/cert-manager-controller:{}".format(data.values.app_version)
+      - image: #@ "docker.io/library/cert-manager-cainjector-amd64:{}".format(data.values.app_version)
diff --git a/test/fixtures/upgrade/overlay/controller-ops.yaml b/test/fixtures/upgrade/overlay/controller-ops.yaml
index 20dc73c9b..6a86debae 100644
--- a/test/fixtures/upgrade/overlay/controller-ops.yaml
+++ b/test/fixtures/upgrade/overlay/controller-ops.yaml
@@ -22,5 +22,5 @@ spec:
     spec:
       containers:
       #@overlay/match by=overlay.subset({"name": "cert-manager"})
-      - image: #@ "quay.io/jetstack/cert-manager-controller:{}".format(data.values.app_version)
+      - image: #@ "docker.io/library/cert-manager-controller-amd64:{}".format(data.values.app_version)
 
diff --git a/test/fixtures/upgrade/overlay/webhook-ops.yaml b/test/fixtures/upgrade/overlay/webhook-ops.yaml
index 31376706b..dea974404 100644
--- a/test/fixtures/upgrade/overlay/webhook-ops.yaml
+++ b/test/fixtures/upgrade/overlay/webhook-ops.yaml
@@ -22,4 +22,4 @@ spec:
     spec:
       containers:
       #@overlay/match by=overlay.subset({"name": "cert-manager"})
-      - image: #@ "quay.io/jetstack/cert-manager-webhook:{}".format(data.values.app_version)
+      - image: #@ "docker.io/library/cert-manager-webhook-amd64:{}".format(data.values.app_version)