From f0234a0868a6c4d0881e5a09af125eacf5b41467 Mon Sep 17 00:00:00 2001 From: James Munnelly Date: Fri, 7 Sep 2018 16:37:25 +0100 Subject: [PATCH 1/6] Add e2e test script utilising 'kind' Signed-off-by: James Munnelly --- Makefile | 6 +- hack/ci/run-e2e-kind.sh | 107 +++++++++++++++++++++++++++++++++ test/fixtures/kind-config.yaml | 32 ++++++++++ 3 files changed, 142 insertions(+), 3 deletions(-) create mode 100755 hack/ci/run-e2e-kind.sh create mode 100644 test/fixtures/kind-config.yaml diff --git a/Makefile b/Makefile index b2729c83c..a8e18e80f 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ PACKAGE_NAME := github.com/jetstack/cert-manager REGISTRY := quay.io/jetstack APP_NAME := cert-manager IMAGE_TAGS := canary -GOPATH ?= $HOME/go +GOPATH ?= $$HOME/go HACK_DIR ?= hack BUILD_TAG := build @@ -24,7 +24,7 @@ BUILD_TAG := build # which require a domain that resolves to the ingress controller to be used for # e2e tests. E2E_NGINX_CERTIFICATE_DOMAIN= - +KUBECONFIG ?= $$HOME/.kube/config PEBBLE_IMAGE_REPO=quay.io/munnerz/pebble # AppVersion is set as the AppVersion to be compiled into the controller binary. @@ -139,7 +139,7 @@ e2e_test: mkdir -p "$$(pwd)/_artifacts" # TODO: make these paths configurable # Run e2e tests - KUBECONFIG=$$HOME/.kube/config CERTMANAGERCONFIG=$$HOME/.kube/config \ + KUBECONFIG=$(KUBECONFIG) CERTMANAGERCONFIG=$(KUBECONFIG) \ ./e2e-tests \ -acme-nginx-certificate-domain=$(E2E_NGINX_CERTIFICATE_DOMAIN) \ -cloudflare-email=$${CLOUDFLARE_E2E_EMAIL} \ diff --git a/hack/ci/run-e2e-kind.sh b/hack/ci/run-e2e-kind.sh new file mode 100755 index 000000000..d4543159f --- /dev/null +++ b/hack/ci/run-e2e-kind.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Copyright 2018 The Jetstack cert-manager contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script will provision an end-to-end testing environment using 'kind' +# (kubernetes-in-docker). +# +# It requires 'kind', 'helm', 'kubectl' and 'docker' to be installed. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") +REPO_ROOT="${SCRIPT_ROOT}/../.." + +KIND_CLUSTER_NAME="cm-e2e" +# TODO: can we rely on this being fixed as such? +KIND_CONTAINER_NAME="kind-${KIND_CLUSTER_NAME}-control-plane" + +# cleanup will call kind delete - it will absorb errors +cleanup() { + # Ignore errors here + kind delete --name="${KIND_CLUSTER_NAME}" || true +} +trap cleanup EXIT + +# deploy_kind will deploy a kubernetes-in-docker cluster +deploy_kind() { + # build kind base and node image + # TODO: use pre-built kind images + kind build base + kind build node --type=apt + + # Create a directory to contain the final KUBECONFIG file + mkdir -p "$HOME/.kube" + + # create the kind cluster + kind create --name="${KIND_CLUSTER_NAME}" --config "${REPO_ROOT}"/test/fixtures/kind-config.yaml + + export KUBECONFIG="${HOME}/.kube/kind-config-${KIND_CLUSTER_NAME}" + + # copy kubectl out of the kind container if kubectl is not installed on the + # host machine. This will *only* work on Linux :this_is_fine: + if ! which kubectl; then + tmp_path=$(mktemp -d) + export PATH="${tmp_path}:${PATH}" + docker cp "${KIND_CONTAINER_NAME}":"$(docker exec "${KIND_CONTAINER_NAME}" which kubectl)" "${tmp_path}/kubectl" + fi + + # Ensure the apiserver is responding + kubectl get nodes +} + +# install_tiller will install tiller with the cluster-admin role bound to its +# service account +install_tiller() { + # Install tiller with admin permissions + kubectl create serviceaccount -n kube-system tiller + # Bind the tiller service account to the cluster-admin role + kubectl create clusterrolebinding tiller-binding --clusterrole=cluster-admin --serviceaccount kube-system:tiller + # Deploy tiller + helm init --service-account tiller --wait +} + +# install_nginx will install nginx-ingress in the cluster and expose it on the +# fixed cluster IP of 10.0.0.15 +install_nginx() { + # Install nginx-ingress with fixed IP + helm install stable/nginx-ingress \ + --name nginx-ingress \ + --namespace kube-system \ + --set controller.service.clusterIP=10.0.0.15 \ + --set controller.service.type=ClusterIP \ + --wait +} + +# build_images will build cert-manager docker images and copy them across to the +# kind docker container running the cluster, so they are available to the +# cluster's docker daemon. +build_images() { + # Build cert-manager binaries & docker image + make build APP_VERSION=build + + docker save quay.io/jetstack/cert-manager-controller:build quay.io/jetstack/cert-manager-acmesolver:build quay.io/jetstack/cert-manager-webhook:build -o cmbundle.tar.gz + docker cp cmbundle.tar.gz "${KIND_CONTAINER_NAME}":/cmbundle.tar.gz + docker exec "${KIND_CONTAINER_NAME}" docker load -i /cmbundle.tar.gz +} + +deploy_kind +install_tiller +install_nginx +build_images + +make e2e_test E2E_NGINX_CERTIFICATE_DOMAIN=certmanager.kubernetes.network KUBECONFIG=${KUBECONFIG} diff --git a/test/fixtures/kind-config.yaml b/test/fixtures/kind-config.yaml new file mode 100644 index 000000000..e7b7dca84 --- /dev/null +++ b/test/fixtures/kind-config.yaml @@ -0,0 +1,32 @@ +# this config file is similar to the default, except we set the cluster's +# service cidr range to be 10.0.0.0/16. +# we do this because we need a fixed/predictable clusterIP of 10.0.0.15 for the +# nginx-ingress service, in order to perform HTTP01 validations during tests. + +apiVersion: kind.sigs.k8s.io/v1alpha1 +kind: Config +# number of nodes in the cluster (currently only 1 is supported) +numNodes: 1 +# template for kubeadm config, "" -> the default template +kubeadmConfigTemplate: | + # config generated by kind + apiVersion: kubeadm.k8s.io/v1alpha2 + kind: MasterConfiguration + clusterName: {{.ClusterName}} + # on docker for mac we have to expose the api server via port forward, + # so we need to ensure the cert is valid for localhost so we can talk + # to the cluster after rewriting the kubeconfig to point to localhost + apiServerCertSANs: [localhost] + kubernetesVersion: {{.KubernetesVersion}} + {{if ne .UnifiedControlPlaneImage ""}} + # optionally specify a unified control plane image + unifiedControlPlaneImage: {{.UnifiedControlPlaneImage}}:{{.DockerStableTag}} + {{end}} + networking: + # Don't think setting pod subnet is currently required + # podSubnet: "" + serviceSubnet: 10.0.0.0/16 + kubeletConfiguration: + baseConfig: + clusterDNS: + - 10.0.0.10 From 8c183d1550f05d3d986a720ca159ef2a7447dad6 Mon Sep 17 00:00:00 2001 From: James Munnelly Date: Fri, 7 Sep 2018 19:25:56 +0100 Subject: [PATCH 2/6] Use prebuilt kind image Signed-off-by: James Munnelly --- hack/ci/run-e2e-kind.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/hack/ci/run-e2e-kind.sh b/hack/ci/run-e2e-kind.sh index d4543159f..1db9b22ba 100755 --- a/hack/ci/run-e2e-kind.sh +++ b/hack/ci/run-e2e-kind.sh @@ -29,6 +29,7 @@ REPO_ROOT="${SCRIPT_ROOT}/../.." KIND_CLUSTER_NAME="cm-e2e" # TODO: can we rely on this being fixed as such? KIND_CONTAINER_NAME="kind-${KIND_CLUSTER_NAME}-control-plane" +KIND_IMAGE=${KIND_IMAGE:-eu.gcr.io/jetstack-build-infra-images/kind:1.11.2-0} # cleanup will call kind delete - it will absorb errors cleanup() { @@ -39,16 +40,11 @@ trap cleanup EXIT # deploy_kind will deploy a kubernetes-in-docker cluster deploy_kind() { - # build kind base and node image - # TODO: use pre-built kind images - kind build base - kind build node --type=apt - - # Create a directory to contain the final KUBECONFIG file - mkdir -p "$HOME/.kube" - # create the kind cluster - kind create --name="${KIND_CLUSTER_NAME}" --config "${REPO_ROOT}"/test/fixtures/kind-config.yaml + kind create \ + --name="${KIND_CLUSTER_NAME}" \ + --image="${KIND_IMAGE}" \ + --config "${REPO_ROOT}"/test/fixtures/kind-config.yaml export KUBECONFIG="${HOME}/.kube/kind-config-${KIND_CLUSTER_NAME}" From 2c3b70428d6b4e935195b3db6c524de22e225122 Mon Sep 17 00:00:00 2001 From: James Munnelly Date: Fri, 7 Sep 2018 19:55:52 +0100 Subject: [PATCH 3/6] Add doc on running end to end tests Signed-off-by: James Munnelly --- docs/devel/end-to-end-tests.rst | 60 +++++++++++++++++++++++++++++++++ docs/devel/index.rst | 1 + 2 files changed, 61 insertions(+) create mode 100644 docs/devel/end-to-end-tests.rst diff --git a/docs/devel/end-to-end-tests.rst b/docs/devel/end-to-end-tests.rst new file mode 100644 index 000000000..181964902 --- /dev/null +++ b/docs/devel/end-to-end-tests.rst @@ -0,0 +1,60 @@ +======================== +Running end-to-end tests +======================== + +cert-manager has an end-to-end test suite that verifies functionality against a +real Kubernetes cluster. + +This document explains how you can run the end-to-end tests yourself. +This is useful when you have added or changed functionality in cert-manager and +want to verify the software still works as expected. + +Requirements +============ + +Currently, a number of tools **must** be installed on your machine in order to +run the tests: + +* ``docker`` - We provision a whole Kubernetes cluster within Docker, and so + an up to date version of Docker must be installed. The oldest Docker version + we have tested is 17.09. + +* kind_ - This + tool is responsible for actually building and starting the Kubernetes cluster + used during tests. + +* helm_ - A minimum version 2.10 is required. + +* ``kubectl`` - If you are running the tests on Linux, this step is + technically not required. For non-Linux hosts (i.e. OSX), you will need to + ensure you have a relatively new version of kubectl available on your PATH. + +* ``golang`` - We require golang to build cert-manager and various test + related components. You should use at least go version 1.9, although we + currently build with go 1.11 in our own CI. + +* An internet connection - tests require access to DNS, and optionally + Cloudflare APIs (if a Cloudflare API token is provided). + +Docker, helm and kubectl should be installed through your preferred means. + +``kind`` can be installed like so: + +.. code-block:: shell + + go install k8s.io/test-infra/kind + +Run end-to-end tests +==================== + +You can run the end-to-end tests by executing the following: + +.. code-block:: shell + + ./hack/ci/run-e2e-kind.sh + +The full suite may take up to 20 minutes to run. +You can monitor output of this command to track progress. + +.. _kind: https://github.com/kubernetes/test-infra/tree/master/kind +.. _helm: https://github.com/helm/helm diff --git a/docs/devel/index.rst b/docs/devel/index.rst index fa4dfc09e..043b02f4d 100644 --- a/docs/devel/index.rst +++ b/docs/devel/index.rst @@ -5,5 +5,6 @@ Development documentation :maxdepth: 1 develop-with-minikube + end-to-end-tests dns01-providers dco-sign-off From 22f4f00ea364002a105fba2d966fcf19676c4000 Mon Sep 17 00:00:00 2001 From: James Munnelly Date: Fri, 7 Sep 2018 21:06:12 +0100 Subject: [PATCH 4/6] Add preKubeadm hook to download latest kubeadm binary Signed-off-by: James Munnelly --- test/fixtures/kind-config.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/fixtures/kind-config.yaml b/test/fixtures/kind-config.yaml index e7b7dca84..e2801aa66 100644 --- a/test/fixtures/kind-config.yaml +++ b/test/fixtures/kind-config.yaml @@ -30,3 +30,20 @@ kubeadmConfigTemplate: | baseConfig: clusterDNS: - 10.0.0.10 +nodeLifecycle: + preKubeadm: + # we download kubeadm 1.11.x here manually as earlier version of kubeadm do + # not support the kubeadm MasterConfiguration structure + - name: download kubeadm 1.11.2 + mustSucceed: true + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + curl -o /usr/bin/kubeadm https://storage.googleapis.com/kubernetes-release/release/v1.11.2/bin/linux/amd64/kubeadm + chmod +x /usr/bin/kubeadm From 399919dffaea91f5af1cadc062d20c13900fc470 Mon Sep 17 00:00:00 2001 From: James Munnelly Date: Fri, 7 Sep 2018 21:51:45 +0100 Subject: [PATCH 5/6] Support older kubernetes version Signed-off-by: James Munnelly --- hack/ci/run-e2e-kind.sh | 8 ++++- test/e2e/e2e.go | 8 +++-- test/fixtures/kind-config-force.yaml | 53 ++++++++++++++++++++++++++++ test/fixtures/kind-config.yaml | 17 --------- 4 files changed, 66 insertions(+), 20 deletions(-) create mode 100644 test/fixtures/kind-config-force.yaml diff --git a/hack/ci/run-e2e-kind.sh b/hack/ci/run-e2e-kind.sh index 1db9b22ba..54497a251 100755 --- a/hack/ci/run-e2e-kind.sh +++ b/hack/ci/run-e2e-kind.sh @@ -40,11 +40,17 @@ trap cleanup EXIT # deploy_kind will deploy a kubernetes-in-docker cluster deploy_kind() { + + local KIND_CONFIG="kind-config.yaml" + if [ ! -z "${KIND_KUBEADM_1_11_FORCE:-}" ]; then + KIND_CONFIG="kind-config-force.yaml" + fi + # create the kind cluster kind create \ --name="${KIND_CLUSTER_NAME}" \ --image="${KIND_IMAGE}" \ - --config "${REPO_ROOT}"/test/fixtures/kind-config.yaml + --config "${REPO_ROOT}"/test/fixtures/"${KIND_CONFIG}" export KUBECONFIG="${HOME}/.kube/kind-config-${KIND_CLUSTER_NAME}" diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 1bfa6da6b..4d4206c43 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -52,11 +52,15 @@ func RunE2ETests(t *testing.T) { } glog.Infof("Installing cert-manager helm chart") - InstallHelmChart(t, releaseName, "./contrib/charts/cert-manager", certManagerDeploymentNamespace, "./test/fixtures/cert-manager-values.yaml") + var extraArgs []string + if os.Getenv("DISABLE_WEBHOOK") == "true" { + extraArgs = append(extraArgs, "--set", "webhook.enabled=false") + } + InstallHelmChart(t, releaseName, "./contrib/charts/cert-manager", certManagerDeploymentNamespace, "./test/fixtures/cert-manager-values.yaml", extraArgs...) glog.Infof("Installing pebble chart") // 10 minute timeout for pebble install due to large images - extraArgs := []string{"--timeout", "600"} + extraArgs = []string{"--timeout", "600"} if framework.TestContext.PebbleImageRepo != "" { extraArgs = append(extraArgs, "--set", "image.repository="+framework.TestContext.PebbleImageRepo) } diff --git a/test/fixtures/kind-config-force.yaml b/test/fixtures/kind-config-force.yaml new file mode 100644 index 000000000..21efaee43 --- /dev/null +++ b/test/fixtures/kind-config-force.yaml @@ -0,0 +1,53 @@ +# this config is a copy of kind-config.yaml, but with an additional lifecycle +# hook that downloads kubeadm 1.11.2 which is able to bootstrap kubernetes 1.10 +# clusters needed during tests. +# +# this config file is similar to the default, except we set the cluster's +# service cidr range to be 10.0.0.0/16. +# we do this because we need a fixed/predictable clusterIP of 10.0.0.15 for the +# nginx-ingress service, in order to perform HTTP01 validations during tests. + +apiVersion: kind.sigs.k8s.io/v1alpha1 +kind: Config +# number of nodes in the cluster (currently only 1 is supported) +numNodes: 1 +# template for kubeadm config, "" -> the default template +kubeadmConfigTemplate: | + # config generated by kind + apiVersion: kubeadm.k8s.io/v1alpha2 + kind: MasterConfiguration + clusterName: {{.ClusterName}} + # on docker for mac we have to expose the api server via port forward, + # so we need to ensure the cert is valid for localhost so we can talk + # to the cluster after rewriting the kubeconfig to point to localhost + apiServerCertSANs: [localhost] + kubernetesVersion: {{.KubernetesVersion}} + {{if ne .UnifiedControlPlaneImage ""}} + # optionally specify a unified control plane image + unifiedControlPlaneImage: {{.UnifiedControlPlaneImage}}:{{.DockerStableTag}} + {{end}} + networking: + # Don't think setting pod subnet is currently required + # podSubnet: "" + serviceSubnet: 10.0.0.0/16 + kubeletConfiguration: + baseConfig: + clusterDNS: + - 10.0.0.10 +nodeLifecycle: + preKubeadm: + # we download kubeadm 1.11.x here manually as earlier version of kubeadm do + # not support the kubeadm MasterConfiguration structure + - name: download kubeadm 1.11.2 + mustSucceed: true + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + curl -o /usr/bin/kubeadm https://storage.googleapis.com/kubernetes-release/release/v1.11.2/bin/linux/amd64/kubeadm + chmod +x /usr/bin/kubeadm diff --git a/test/fixtures/kind-config.yaml b/test/fixtures/kind-config.yaml index e2801aa66..e7b7dca84 100644 --- a/test/fixtures/kind-config.yaml +++ b/test/fixtures/kind-config.yaml @@ -30,20 +30,3 @@ kubeadmConfigTemplate: | baseConfig: clusterDNS: - 10.0.0.10 -nodeLifecycle: - preKubeadm: - # we download kubeadm 1.11.x here manually as earlier version of kubeadm do - # not support the kubeadm MasterConfiguration structure - - name: download kubeadm 1.11.2 - mustSucceed: true - command: - - /bin/bash - - -c - - | - #!/bin/bash - set -o errexit - set -o nounset - set -o pipefail - - curl -o /usr/bin/kubeadm https://storage.googleapis.com/kubernetes-release/release/v1.11.2/bin/linux/amd64/kubeadm - chmod +x /usr/bin/kubeadm From 5005c1723b6d7753c0a22bd053492cb3e1dc27fd Mon Sep 17 00:00:00 2001 From: James Munnelly Date: Fri, 7 Sep 2018 23:03:12 +0100 Subject: [PATCH 6/6] Use temporary directory for cmbundle Signed-off-by: James Munnelly --- hack/ci/run-e2e-kind.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hack/ci/run-e2e-kind.sh b/hack/ci/run-e2e-kind.sh index 54497a251..c176f6da7 100755 --- a/hack/ci/run-e2e-kind.sh +++ b/hack/ci/run-e2e-kind.sh @@ -18,6 +18,7 @@ # (kubernetes-in-docker). # # It requires 'kind', 'helm', 'kubectl' and 'docker' to be installed. +# kubectl will be automatically installed if not found when on linux set -o errexit set -o nounset @@ -96,9 +97,12 @@ build_images() { # Build cert-manager binaries & docker image make build APP_VERSION=build - docker save quay.io/jetstack/cert-manager-controller:build quay.io/jetstack/cert-manager-acmesolver:build quay.io/jetstack/cert-manager-webhook:build -o cmbundle.tar.gz - docker cp cmbundle.tar.gz "${KIND_CONTAINER_NAME}":/cmbundle.tar.gz + local TMP_DIR=$(mktemp -d) + local BUNDLE_FILE="${TMP_DIR}"/cmbundle.tar.gz + docker save quay.io/jetstack/cert-manager-controller:build quay.io/jetstack/cert-manager-acmesolver:build quay.io/jetstack/cert-manager-webhook:build -o "${BUNDLE_FILE}" + docker cp "${BUNDLE_FILE}" "${KIND_CONTAINER_NAME}":/cmbundle.tar.gz docker exec "${KIND_CONTAINER_NAME}" docker load -i /cmbundle.tar.gz + rm -Rf "${TMP_DIR}" } deploy_kind