Merge pull request #876 from munnerz/kind-builds

Use kind for e2e tests and add e2e testing documentation

Makefile

@@ -16,7 +16,7 @@ PACKAGE_NAME := github.com/jetstack/cert-manager
 REGISTRY := quay.io/jetstack
 APP_NAME := cert-manager
 IMAGE_TAGS := canary
-GOPATH ?= $HOME/go
+GOPATH ?= $$HOME/go
 HACK_DIR ?= hack
 BUILD_TAG := build
 
@@ -24,7 +24,7 @@ BUILD_TAG := build
 # which require a domain that resolves to the ingress controller to be used for
 # e2e tests.
 E2E_NGINX_CERTIFICATE_DOMAIN=
-
+KUBECONFIG ?= $$HOME/.kube/config
 PEBBLE_IMAGE_REPO=quay.io/munnerz/pebble
 
 # AppVersion is set as the AppVersion to be compiled into the controller binary.
@@ -139,7 +139,7 @@ e2e_test:
 	mkdir -p "$$(pwd)/_artifacts"
 	# TODO: make these paths configurable
 	# Run e2e tests
-	KUBECONFIG=$$HOME/.kube/config CERTMANAGERCONFIG=$$HOME/.kube/config \
+	KUBECONFIG=$(KUBECONFIG) CERTMANAGERCONFIG=$(KUBECONFIG) \
 		./e2e-tests \
 			-acme-nginx-certificate-domain=$(E2E_NGINX_CERTIFICATE_DOMAIN) \
 			-cloudflare-email=$${CLOUDFLARE_E2E_EMAIL} \

docs/devel/end-to-end-tests.rst

@@ -0,0 +1,60 @@
+========================
+Running end-to-end tests
+========================
+
+cert-manager has an end-to-end test suite that verifies functionality against a
+real Kubernetes cluster.
+
+This document explains how you can run the end-to-end tests yourself.
+This is useful when you have added or changed functionality in cert-manager and
+want to verify the software still works as expected.
+
+Requirements
+============
+
+Currently, a number of tools **must** be installed on your machine in order to
+run the tests:
+
+* ``docker`` - We provision a whole Kubernetes cluster within Docker, and so
+  an up to date version of Docker must be installed. The oldest Docker version
+  we have tested is 17.09.
+
+* kind_ - This
+  tool is responsible for actually building and starting the Kubernetes cluster
+  used during tests.
+
+* helm_ - A minimum version 2.10 is required.
+
+* ``kubectl`` - If you are running the tests on Linux, this step is
+  technically not required. For non-Linux hosts (i.e. OSX), you will need to
+  ensure you have a relatively new version of kubectl available on your PATH.
+
+* ``golang`` - We require golang to build cert-manager and various test
+  related components. You should use at least go version 1.9, although we
+  currently build with go 1.11 in our own CI.
+
+* An internet connection - tests require access to DNS, and optionally
+  Cloudflare APIs (if a Cloudflare API token is provided).
+
+Docker, helm and kubectl should be installed through your preferred means.
+
+``kind`` can be installed like so:
+
+.. code-block:: shell
+
+   go install k8s.io/test-infra/kind
+
+Run end-to-end tests
+====================
+
+You can run the end-to-end tests by executing the following:
+
+.. code-block:: shell
+
+   ./hack/ci/run-e2e-kind.sh
+
+The full suite may take up to 20 minutes to run.
+You can monitor output of this command to track progress.
+
+.. _kind: https://github.com/kubernetes/test-infra/tree/master/kind
+.. _helm: https://github.com/helm/helm

docs/devel/index.rst

@@ -5,5 +5,6 @@ Development documentation
    :maxdepth: 1
 
    develop-with-minikube
+   end-to-end-tests
    dns01-providers
    dco-sign-off

hack/ci/run-e2e-kind.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+# Copyright 2018 The Jetstack cert-manager contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script will provision an end-to-end testing environment using 'kind'
+# (kubernetes-in-docker).
+#
+# It requires 'kind', 'helm', 'kubectl' and 'docker' to be installed.
+# kubectl will be automatically installed if not found when on linux
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")
+REPO_ROOT="${SCRIPT_ROOT}/../.."
+
+KIND_CLUSTER_NAME="cm-e2e"
+# TODO: can we rely on this being fixed as such?
+KIND_CONTAINER_NAME="kind-${KIND_CLUSTER_NAME}-control-plane"
+KIND_IMAGE=${KIND_IMAGE:-eu.gcr.io/jetstack-build-infra-images/kind:1.11.2-0}
+
+# cleanup will call kind delete - it will absorb errors
+cleanup() {
+    # Ignore errors here
+    kind delete --name="${KIND_CLUSTER_NAME}" || true
+}
+trap cleanup EXIT
+
+# deploy_kind will deploy a kubernetes-in-docker cluster
+deploy_kind() {
+
+    local KIND_CONFIG="kind-config.yaml"
+    if [ ! -z "${KIND_KUBEADM_1_11_FORCE:-}" ]; then
+        KIND_CONFIG="kind-config-force.yaml"
+    fi
+
+    # create the kind cluster
+    kind create \
+        --name="${KIND_CLUSTER_NAME}" \
+        --image="${KIND_IMAGE}" \
+        --config "${REPO_ROOT}"/test/fixtures/"${KIND_CONFIG}"
+
+    export KUBECONFIG="${HOME}/.kube/kind-config-${KIND_CLUSTER_NAME}"
+
+    # copy kubectl out of the kind container if kubectl is not installed on the
+    # host machine. This will *only* work on Linux :this_is_fine:
+    if ! which kubectl; then
+        tmp_path=$(mktemp -d)
+        export PATH="${tmp_path}:${PATH}"
+        docker cp "${KIND_CONTAINER_NAME}":"$(docker exec "${KIND_CONTAINER_NAME}" which kubectl)" "${tmp_path}/kubectl"
+    fi
+
+    # Ensure the apiserver is responding
+    kubectl get nodes
+}
+
+# install_tiller will install tiller with the cluster-admin role bound to its
+# service account
+install_tiller() {
+    # Install tiller with admin permissions
+    kubectl create serviceaccount -n kube-system tiller
+    # Bind the tiller service account to the cluster-admin role
+    kubectl create clusterrolebinding tiller-binding --clusterrole=cluster-admin --serviceaccount kube-system:tiller
+    # Deploy tiller
+    helm init --service-account tiller --wait
+}
+
+# install_nginx will install nginx-ingress in the cluster and expose it on the
+# fixed cluster IP of 10.0.0.15
+install_nginx() {
+    # Install nginx-ingress with fixed IP
+    helm install stable/nginx-ingress \
+        --name nginx-ingress \
+        --namespace kube-system \
+        --set controller.service.clusterIP=10.0.0.15 \
+        --set controller.service.type=ClusterIP \
+        --wait
+}
+
+# build_images will build cert-manager docker images and copy them across to the
+# kind docker container running the cluster, so they are available to the
+# cluster's docker daemon.
+build_images() {
+    # Build cert-manager binaries & docker image
+    make build APP_VERSION=build
+
+    local TMP_DIR=$(mktemp -d)
+    local BUNDLE_FILE="${TMP_DIR}"/cmbundle.tar.gz
+    docker save quay.io/jetstack/cert-manager-controller:build quay.io/jetstack/cert-manager-acmesolver:build quay.io/jetstack/cert-manager-webhook:build -o "${BUNDLE_FILE}"
+    docker cp "${BUNDLE_FILE}" "${KIND_CONTAINER_NAME}":/cmbundle.tar.gz
+    docker exec "${KIND_CONTAINER_NAME}" docker load -i /cmbundle.tar.gz
+    rm -Rf "${TMP_DIR}"
+}
+
+deploy_kind
+install_tiller
+install_nginx
+build_images
+
+make e2e_test E2E_NGINX_CERTIFICATE_DOMAIN=certmanager.kubernetes.network KUBECONFIG=${KUBECONFIG}

test/e2e/e2e.go

@@ -52,11 +52,15 @@ func RunE2ETests(t *testing.T) {
 	}
 
 	glog.Infof("Installing cert-manager helm chart")
-	InstallHelmChart(t, releaseName, "./contrib/charts/cert-manager", certManagerDeploymentNamespace, "./test/fixtures/cert-manager-values.yaml")
+	var extraArgs []string
+	if os.Getenv("DISABLE_WEBHOOK") == "true" {
+		extraArgs = append(extraArgs, "--set", "webhook.enabled=false")
+	}
+	InstallHelmChart(t, releaseName, "./contrib/charts/cert-manager", certManagerDeploymentNamespace, "./test/fixtures/cert-manager-values.yaml", extraArgs...)
 
 	glog.Infof("Installing pebble chart")
 	// 10 minute timeout for pebble install due to large images
-	extraArgs := []string{"--timeout", "600"}
+	extraArgs = []string{"--timeout", "600"}
 	if framework.TestContext.PebbleImageRepo != "" {
 		extraArgs = append(extraArgs, "--set", "image.repository="+framework.TestContext.PebbleImageRepo)
 	}

test/fixtures/kind-config-force.yaml

@@ -0,0 +1,53 @@
+# this config is a copy of kind-config.yaml, but with an additional lifecycle
+# hook that downloads kubeadm 1.11.2 which is able to bootstrap kubernetes 1.10
+# clusters needed during tests.
+#
+# this config file is similar to the default, except we set the cluster's
+# service cidr range to be 10.0.0.0/16.
+# we do this because we need a fixed/predictable clusterIP of 10.0.0.15 for the
+# nginx-ingress service, in order to perform HTTP01 validations during tests.
+
+apiVersion: kind.sigs.k8s.io/v1alpha1
+kind: Config
+# number of nodes in the cluster (currently only 1 is supported)
+numNodes: 1
+# template for kubeadm config, "" -> the default template
+kubeadmConfigTemplate: |
+  # config generated by kind
+  apiVersion: kubeadm.k8s.io/v1alpha2
+  kind: MasterConfiguration
+  clusterName: {{.ClusterName}}
+  # on docker for mac we have to expose the api server via port forward,
+  # so we need to ensure the cert is valid for localhost so we can talk
+  # to the cluster after rewriting the kubeconfig to point to localhost
+  apiServerCertSANs: [localhost]
+  kubernetesVersion: {{.KubernetesVersion}}
+  {{if ne .UnifiedControlPlaneImage ""}}
+  # optionally specify a unified control plane image
+  unifiedControlPlaneImage: {{.UnifiedControlPlaneImage}}:{{.DockerStableTag}}
+  {{end}}
+  networking:
+    # Don't think setting pod subnet is currently required
+    # podSubnet: ""
+    serviceSubnet: 10.0.0.0/16
+  kubeletConfiguration:
+    baseConfig:
+      clusterDNS:
+      - 10.0.0.10
+nodeLifecycle:
+  preKubeadm:
+  # we download kubeadm 1.11.x here manually as earlier version of kubeadm do
+  # not support the kubeadm MasterConfiguration structure
+  - name: download kubeadm 1.11.2
+    mustSucceed: true
+    command:
+    - /bin/bash
+    - -c
+    - |
+      #!/bin/bash
+      set -o errexit
+      set -o nounset
+      set -o pipefail
+
+      curl -o /usr/bin/kubeadm https://storage.googleapis.com/kubernetes-release/release/v1.11.2/bin/linux/amd64/kubeadm
+      chmod +x /usr/bin/kubeadm

test/fixtures/kind-config.yaml

@@ -0,0 +1,32 @@
+# this config file is similar to the default, except we set the cluster's
+# service cidr range to be 10.0.0.0/16.
+# we do this because we need a fixed/predictable clusterIP of 10.0.0.15 for the
+# nginx-ingress service, in order to perform HTTP01 validations during tests.
+
+apiVersion: kind.sigs.k8s.io/v1alpha1
+kind: Config
+# number of nodes in the cluster (currently only 1 is supported)
+numNodes: 1
+# template for kubeadm config, "" -> the default template
+kubeadmConfigTemplate: |
+  # config generated by kind
+  apiVersion: kubeadm.k8s.io/v1alpha2
+  kind: MasterConfiguration
+  clusterName: {{.ClusterName}}
+  # on docker for mac we have to expose the api server via port forward,
+  # so we need to ensure the cert is valid for localhost so we can talk
+  # to the cluster after rewriting the kubeconfig to point to localhost
+  apiServerCertSANs: [localhost]
+  kubernetesVersion: {{.KubernetesVersion}}
+  {{if ne .UnifiedControlPlaneImage ""}}
+  # optionally specify a unified control plane image
+  unifiedControlPlaneImage: {{.UnifiedControlPlaneImage}}:{{.DockerStableTag}}
+  {{end}}
+  networking:
+    # Don't think setting pod subnet is currently required
+    # podSubnet: ""
+    serviceSubnet: 10.0.0.0/16
+  kubeletConfiguration:
+    baseConfig:
+      clusterDNS:
+      - 10.0.0.10