add make-based upgrade test

This uses cmctl instead of kubectl_cert-manager, uses make instead of bazel and fixes an incorrect container name in test/fixtures/upgrade/overlay/cainjector-ops.yaml Signed-off-by: Ashley Davis <ashley.davis@jetstack.io>
2022-06-30 16:51:00 +01:00 · 2022-06-30 16:51:00 +01:00 · ca35696244
commit ca35696244
parent 27e1a770b3
7 changed files with 98 additions and 98 deletions
--- a/hack/verify-upgrade.sh
+++ b/hack/verify-upgrade.sh
@ -20,61 +20,59 @@ set -o pipefail

 SCRIPT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )"
 export REPO_ROOT="${SCRIPT_ROOT}/.."
-source "${REPO_ROOT}/devel/lib/lib.sh"
 source "${REPO_ROOT}/hack/build/version.sh"
-export APP_VERSION="$(date +"%s")"

 kube::version::last_published_release

 LATEST_RELEASE="${KUBE_LAST_RELEASE}"

-# Ensure helm, kind, kubectl, ytt are available
-echo "Building the required tools.."
-bazel build //hack/bin:helm //hack/bin:kind //hack/bin:ytt //hack/bin:kubectl //hack/bin:kubectl-cert_manager
-bindir="$(bazel info bazel-bin)"
-export PATH="${bindir}/hack/bin/:$PATH"
+usage_and_exit() {
+	echo "usage: $0 <path-to-helm> <path-to-kind> <path-to-ytt> <path-to-kubectl> <path-to-cmctl>" >&2
+	exit 1
+}

-# Build images from latest master and load into the kind cluster. These will be
-# used when upgrading with both kubectl and helm.
-# Tag images with APP_VERSION for consistency with devel/addon/certmanager/install.sh.
-echo "Building latest cert-manger images.."
-APP_VERSION=${APP_VERSION} \
-bazel run \
-  --stamp=true \
-  --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \
-  "//devel/addon/certmanager:bundle"
+if [[ -z "${1:-}" || -z "${2:-}" || -z "${3:-}" ||-z "${4:-}" || -z "${5:-}" ]]; then
+	usage_and_exit
+fi

-echo "Loading latest cert-manager images to cluster.."
-load_image "quay.io/jetstack/cert-manager-controller:${APP_VERSION}" &
-load_image "quay.io/jetstack/cert-manager-acmesolver:${APP_VERSION}" &
-load_image "quay.io/jetstack/cert-manager-cainjector:${APP_VERSION}" &
-load_image "quay.io/jetstack/cert-manager-webhook:${APP_VERSION}" &
-load_image "quay.io/jetstack/cert-manager-ctl:${APP_VERSION}" &
-wait
+helm=$(realpath "$1")
+kind=$(realpath "$2")
+ytt=$(realpath "$3")
+kubectl=$(realpath "$4")
+cmctl=$(realpath "$5")

-# Namespace to deploy into
+# Set up a fresh kind cluster
+
+$kind delete clusters kind || :
+make e2e-setup-kind
+
+################################################
+# VERIFY INSTALL, UPGRADE, UNINSTALL WITH HELM #
+################################################
+
+# Namespace we'll deploy into
 NAMESPACE="${NAMESPACE:-cert-manager}"
+
 # Release name to use with Helm
 RELEASE_NAME="${RELEASE_NAME:-cert-manager}"
-# cert-manager Helm chart
-HELM_CHART="jetstack/cert-manager"

-############
-# VERIFY INSTALL, UPGRADE, UNINSTALL WITH HELM
-############
+HELM_URL="https://charts.jetstack.io"

-echo "Testing upgrade from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} with Helm.."
+# cert-manager Helm chart location
+HELM_CHART="cmupgradetest/cert-manager"
+
+echo "+++ Testing upgrading from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} with Helm"

 # This will target the host's helm repository cache
-helm repo add jetstack https://charts.jetstack.io
-helm repo update
+$helm repo add cmupgradetest $HELM_URL
+$helm repo update

 # 1. INSTALL THE LATEST PUBLISHED HELM CHART

-echo "Installing cert-manager ${LATEST_RELEASE} Helm chart into the cluster..."
+echo "+++ Installing cert-manager ${LATEST_RELEASE} Helm chart into the cluster..."

 # Upgrade or install latest published cert-manager Helm release
-helm upgrade \
+$helm upgrade \
    --install \
    --wait \
    --namespace "${NAMESPACE}" \
@ -85,102 +83,98 @@ helm upgrade \
    "$HELM_CHART"

 # Wait for the cert-manager api to be available
-kubectl cert-manager check api --wait=2m -v
+$cmctl check api --wait=2m -v

-echo "Creating some cert-manager resources.."
+echo "+++ Creating some cert-manager resources.."

 # Create a cert-manager issuer and cert
-kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first"
+$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first"

 # Ensure cert becomes ready
-kubectl wait --for=condition=Ready cert/test1 --timeout=180s
+$kubectl wait --for=condition=Ready cert/test1 --timeout=180s

 # 2. BUILD AND UPGRADE TO HELM CHART FROM THE CURRENT MASTER

-bazel build //deploy/charts/cert-manager
-
-echo "Upgrading cert-manager Helm release to commit ${KUBE_GIT_COMMIT}..."
-helm upgrade \
-    --install \
-    --wait \
-    --namespace "${NAMESPACE}" \
-    --set installCRDs=true \
-    --create-namespace \
-    "$RELEASE_NAME" \
-    "$REPO_ROOT/bazel-bin/deploy/charts/cert-manager/cert-manager.tgz"
+# e2e-setup-certamanager both builds and deploys the latest available chart based on the current checkout
+make e2e-setup-certmanager

 # Wait for the cert-manager api to be available
-kubectl cert-manager check api --wait=2m -v
+$cmctl check api --wait=2m -v

 # Test that the existing cert-manager resources can still be retrieved
-kubectl get issuer/selfsigned-issuer cert/test1
+$kubectl get issuer/selfsigned-issuer cert/test1

-echo "Creating some cert-manager resources.."
+echo "+++ Creating some more cert-manager resources.."

-# # Create another certificate
-kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second"
+# Create another certificate
+$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second"

 # Ensure cert becomes ready
-kubectl wait --for=condition=Ready cert/test2 --timeout=180s
+$kubectl wait --for=condition=Ready cert/test2 --timeout=180s

 # 3. UNINSTALL HELM RELEASE
-echo "Uninstalling the Helm release.."

-kubectl delete \
-	-f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml"
+echo "+++ Uninstalling the Helm release"

-helm uninstall \
+$kubectl delete -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml"
+
+$helm uninstall \
    --namespace "${NAMESPACE}" \
    "$RELEASE_NAME"

-kubectl delete "namespace/${NAMESPACE}" \
-	--wait
+$kubectl delete "namespace/${NAMESPACE}" --wait

-############
-# VERIFY INSTALL, UPGRADE, UNINSTALL WITH STATIC MANIFESTS
-############
+############################################################
+# VERIFY INSTALL, UPGRADE, UNINSTALL WITH STATIC MANIFESTS #
+############################################################

 # 1. INSTALL THE LATEST PUBLISHED RELEASE WITH STATIC MANIFESTS

-echo "Testing cert-manager upgrade from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} with static manifests.."
+echo "+++ Testing cert-manager upgrade from ${LATEST_RELEASE} to commit ${KUBE_GIT_COMMIT} using static manifests"

-echo "Install cert-manager ${LATEST_RELEASE} using static manifests.."
-kubectl apply \
+echo "+++ Installing cert-manager ${LATEST_RELEASE} using static manifests"
+
+$kubectl apply \
 	-f "https://github.com/cert-manager/cert-manager/releases/download/${LATEST_RELEASE}/cert-manager.yaml" \
 	--wait

-kubectl wait \
+$kubectl wait \
 	--for=condition=available \
 	--timeout=180s deployment/cert-manager-webhook \
 	--namespace "${NAMESPACE}"

 # Wait for the cert-manager api to be available
-kubectl cert-manager check api --wait=2m -v
+$cmctl check api --wait=2m -v

 # Create a cert-manager issuer and cert
-kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first"
+$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="first"

 # Ensure cert becomes ready
-kubectl wait --for=condition=Ready cert/test1 --timeout=180s
+$kubectl wait --for=condition=Ready cert/test1 --timeout=180s

 # 2. VERIFY UPGRADE TO THE LATEST BUILD FROM MASTER

-echo "Install cert-manager commit ${KUBE_GIT_COMMIT} using static manifests.."
+MANIFEST_LOCATION=${REPO_ROOT}/_bin/yaml/cert-manager.yaml
+
+echo "+++ Installing cert-manager commit ${KUBE_GIT_COMMIT} using static manifests"

 # Build the static manifests
-bazel build //deploy/manifests
+make release-manifests
+
+RELEASE_VERSION=$(make --silent release-version)

 # Overwrite image tags in the static manifests and deploy.
-ytt -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/controller-ops.yaml" \
-	-f "${REPO_ROOT}/test/fixtures/upgrade/overlay/cainjector-ops.yaml" \
-	-f "${REPO_ROOT}/test/fixtures/upgrade/overlay/webhook-ops.yaml" \
-	-f "${REPO_ROOT}/test/fixtures/upgrade/overlay/values.yaml" \
-	-f "${REPO_ROOT}/bazel-bin/deploy/manifests/cert-manager.yaml" \
-	--data-value app_version="${APP_VERSION}" \
-	--ignore-unknown-comments | kubectl apply -f -
+$ytt -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/controller-ops.yaml" \
+     -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/cainjector-ops.yaml" \
+     -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/webhook-ops.yaml" \
+     -f "${REPO_ROOT}/test/fixtures/upgrade/overlay/values.yaml" \
+     -f $MANIFEST_LOCATION \
+     --data-value app_version="${RELEASE_VERSION}" \
+     --ignore-unknown-comments | kubectl apply -f -

-rollout_cmd="kubectl rollout status deployment/cert-manager-webhook --namespace ${NAMESPACE}"
+rollout_cmd="$kubectl rollout status deployment/cert-manager-webhook --namespace ${NAMESPACE}"
 attempts=0
+
 until $rollout_cmd; do
  $rollout_cmd
  ((attempts++))
@ -192,22 +186,21 @@ until $rollout_cmd; do
 done

 # Wait for the cert-manager api to be available
-kubectl cert-manager check api --wait=2m -v
+$cmctl check api --wait=2m -v

 # Test that the existing cert-manager resources can still be retrieved
-kubectl get issuer/selfsigned-issuer cert/test1
+$kubectl get issuer/selfsigned-issuer cert/test1

-echo "Creating some cert-manager resources.."
+echo "+++ Creating some cert-manager resources"

-# # Create another certificate
-kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second"
+# Create another certificate
+$kubectl apply -f "${REPO_ROOT}/test/fixtures/cert-manager-resources.yaml" --selector=test="second"

 # Ensure cert becomes ready
-kubectl wait --for=condition=Ready cert/test2 --timeout=180s
-
-echo "Uninstalling cert-manager.."
+$kubectl wait --for=condition=Ready cert/test2 --timeout=180s

 # 3. UNINSTALL
-kubectl delete \
-	-f "${REPO_ROOT}/bazel-bin/deploy/manifests/cert-manager.yaml" \
-	--wait \
+
+echo "+++ Uninstalling cert-manager"
+
+$kubectl delete -f $MANIFEST_LOCATION --wait
--- a/make/ci.mk
+++ b/make/ci.mk
@ -112,9 +112,8 @@ verify_chart:
 	./hack/verify-chart-version.sh bazel-bin/deploy/charts/cert-manager/cert-manager.tgz

 .PHONY: verify_upgrade
-verify_upgrade:
-	$(warning "The 'verify_upgrade' target is deprecated and will be removed soon. Please use instead 'make e2e-setup-kind && ./hack/verify-upgrade.sh'.")
-	./hack/verify-upgrade.sh
+verify_upgrade: test-upgrade
+	$(warning "The 'verify_upgrade' target is deprecated and will be removed soon. Please use instead 'make test-upgrade'.")

 .PHONY: cluster
 cluster:
--- a/make/git.mk
+++ b/make/git.mk
@ -13,6 +13,10 @@ gitver:
 	@echo "Is prerelease:     \"$(IS_PRERELEASE)\""
 	@echo "Git commit hash:   \"$(GITCOMMIT)\""

+.PHONY: release-version
+release-version:
+	@echo "$(RELEASE_VERSION)"
+
 # Lists all remote tags on the upstream, which gives tags in format:
 # "<commit> ref/tags/<tag>". Strips commit + tag prefix, filters out tags for v1+,
 # and manually removes v1.2.0-alpha.1, since that version's manifest contains
--- a/make/test.mk
+++ b/make/test.mk
@ -75,6 +75,10 @@ e2e: $(BINDIR)/scratch/kind-exists $(BINDIR)/tools/kubectl $(BINDIR)/tools/ginkg
 e2e-ci: e2e-setup-kind e2e-setup
 	$(MAKE) --no-print-directory e2e FLAKE_ATTEMPTS=2 K8S_VERSION="$(K8S_VERSION)" || ($(MAKE) kind-logs && exit 1)

+.PHONY: test-upgrade
+test-upgrade: | $(BINDIR)/tools/helm $(BINDIR)/tools/kind $(BINDIR)/tools/ytt $(BINDIR)/tools/kubectl $(BINDIR)/cmctl/cmctl-$(HOST_OS)-$(HOST_ARCH)
+	./hack/verify-upgrade.sh $(BINDIR)/tools/helm $(BINDIR)/tools/kind $(BINDIR)/tools/ytt $(BINDIR)/tools/kubectl $(BINDIR)/cmctl/cmctl-$(HOST_OS)-$(HOST_ARCH)
+
 test/integration/versionchecker/testdata/test_manifests.tar: $(BINDIR)/scratch/oldcrds.tar $(BINDIR)/yaml/cert-manager.yaml
 	@# Remove the temp files if they exist
 	rm -f $(BINDIR)/scratch/versionchecker-test-manifests.tar $(BINDIR)/scratch/$(RELEASE_VERSION).yaml
--- a/test/fixtures/upgrade/overlay/cainjector-ops.yaml
+++ b/test/fixtures/upgrade/overlay/cainjector-ops.yaml
@ -22,4 +22,4 @@ spec:
    spec:
      containers:
      #@overlay/match by=overlay.subset({"name": "cert-manager"})
-      - image: #@ "quay.io/jetstack/cert-manager-controller:{}".format(data.values.app_version)
+      - image: #@ "docker.io/library/cert-manager-cainjector-amd64:{}".format(data.values.app_version)
--- a/test/fixtures/upgrade/overlay/controller-ops.yaml
+++ b/test/fixtures/upgrade/overlay/controller-ops.yaml
@ -22,5 +22,5 @@ spec:
    spec:
      containers:
      #@overlay/match by=overlay.subset({"name": "cert-manager"})
-      - image: #@ "quay.io/jetstack/cert-manager-controller:{}".format(data.values.app_version)
+      - image: #@ "docker.io/library/cert-manager-controller-amd64:{}".format(data.values.app_version)

--- a/test/fixtures/upgrade/overlay/webhook-ops.yaml
+++ b/test/fixtures/upgrade/overlay/webhook-ops.yaml
@ -22,4 +22,4 @@ spec:
    spec:
      containers:
      #@overlay/match by=overlay.subset({"name": "cert-manager"})
-      - image: #@ "quay.io/jetstack/cert-manager-webhook:{}".format(data.values.app_version)
+      - image: #@ "docker.io/library/cert-manager-webhook-amd64:{}".format(data.values.app_version)