Add WaitForAllPodsRunningInNamespace function as an addon helper

Signed-off-by: James Munnelly <james@munnelly.eu>

test/e2e/framework/BUILD.bazel

@@ -15,6 +15,8 @@ go_library(
         "//pkg/client/clientset/versioned:go_default_library",
         "//test/e2e/framework/addon:go_default_library",
         "//test/e2e/framework/config:go_default_library",
+        "//test/e2e/framework/helper:go_default_library",
+        "//test/e2e/framework/log:go_default_library",
         "//test/e2e/framework/util:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
@@ -44,6 +46,8 @@ filegroup(
         ":package-srcs",
         "//test/e2e/framework/addon:all-srcs",
         "//test/e2e/framework/config:all-srcs",
+        "//test/e2e/framework/helper:all-srcs",
+        "//test/e2e/framework/log:all-srcs",
         "//test/e2e/framework/util:all-srcs",
     ],
     tags = ["automanaged"],

test/e2e/framework/addon/base/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//test/e2e/framework/config:go_default_library",
+        "//test/e2e/framework/helper:go_default_library",
         "//test/e2e/framework/util:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
     ],

test/e2e/framework/addon/base/base.go

@@ -19,6 +19,7 @@ limitations under the License.
 package base
 
 import (
+	"github.com/jetstack/cert-manager/test/e2e/framework/helper"
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/jetstack/cert-manager/test/e2e/framework/config"
@@ -42,6 +43,12 @@ type Details struct {
 	KubeClient kubernetes.Interface
 }
 
+func (d *Details) Helper() *helper.Helper {
+	return &helper.Helper{
+		KubeClient: d.KubeClient,
+	}
+}
+
 func (b *Base) Setup(c *config.Config) error {
 	kubeConfig, err := util.LoadConfig(c.KubeConfig, c.KubeContext)
 	if err != nil {

test/e2e/framework/addon/chart/addon.go

@@ -123,6 +123,11 @@ func (c *Chart) Provision() error {
 		return fmt.Errorf("error install helm chart: %v", err)
 	}
 
+	err = c.Tiller.Base.Details().Helper().WaitForAllPodsRunningInNamespace(c.Namespace)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 

test/e2e/framework/addon/tiller/addon.go

@@ -18,7 +18,6 @@ package tiller
 
 import (
 	"fmt"
-	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -254,32 +253,9 @@ func (t *Tiller) Provision() error {
 	}
 
 	// otherwise lookup the newly created pods name
-	kubeClient := t.Base.Details().KubeClient
-	retries := 10
-	for {
-		pods, err := kubeClient.CoreV1().Pods(t.Namespace).List(metav1.ListOptions{
-			LabelSelector: "name=tiller",
-		})
-		if err != nil {
-			return err
-		}
-		if len(pods.Items) == 0 {
-			if retries == 0 {
-				return fmt.Errorf("failed to create tiller pod within 10s")
-			}
-			retries--
-			time.Sleep(time.Second * 2)
-			continue
-		}
-		tillerPod := pods.Items[0]
-		// If the vault pod exists but is just waiting to be created, we allow
-		// it a bit longer.
-		if len(tillerPod.Status.ContainerStatuses) == 0 || !tillerPod.Status.ContainerStatuses[0].Ready {
-			retries--
-			time.Sleep(time.Second * 5)
-			continue
-		}
-		break
+	err = t.Base.Details().Helper().WaitForAllPodsRunningInNamespace(t.Namespace)
+	if err != nil {
+		return err
 	}
 
 	return nil

test/e2e/framework/framework.go

@@ -17,6 +17,7 @@ limitations under the License.
 package framework
 
 import (
+	"github.com/jetstack/cert-manager/test/e2e/framework/helper"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/api/core/v1"
@@ -157,6 +158,12 @@ func (f *Framework) RequireAddon(a addon.Addon) {
 	})
 }
 
+func (f *Framework) Helper() *helper.Helper {
+	return &helper.Helper{
+		KubeClient: f.KubeClientSet,
+	}
+}
+
 // CertManagerDescribe is a wrapper function for ginkgo describe. Adds namespacing.
 func CertManagerDescribe(text string, body func()) bool {
 	return Describe("[cert-manager] "+text, body)

test/e2e/framework/helper/BUILD.bazel

@@ -0,0 +1,34 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "helper.go",
+        "pod_start.go",
+    ],
+    importpath = "github.com/jetstack/cert-manager/test/e2e/framework/helper",
+    tags = ["manual"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//test/e2e/framework/log:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/framework/helper/helper.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2018 The Jetstack cert-manager contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helper
+
+import "k8s.io/client-go/kubernetes"
+
+// Helper provides methods for common operations needed during tests.
+type Helper struct {
+	KubeClient kubernetes.Interface
+}

test/e2e/framework/helper/pod_start.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2018 The Jetstack cert-manager contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helper
+
+import (
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	. "github.com/jetstack/cert-manager/test/e2e/framework/log"
+)
+
+const (
+	// Poll is how often the API is polled in Wait operations by default
+	Poll = time.Second * 2
+
+	// PodStartTimeout is the default amount of time to wait in pod start operations
+	PodStartTimeout = time.Minute
+)
+
+// WaitForAllPodsRunningInNamespace waits default amount of time (PodStartTimeout)
+// for all pods in the specified namespace to become running.
+func (h *Helper) WaitForAllPodsRunningInNamespace(ns string) error {
+	return h.WaitForAllPodsRunningInNamespaceTimeout(ns, PodStartTimeout)
+}
+
+func (h *Helper) WaitForAllPodsRunningInNamespaceTimeout(ns string, timeout time.Duration) error {
+	By("Waiting " + timeout.String() + " for all pods in namespace '" + ns + "' to be Ready")
+	return wait.PollImmediate(Poll, timeout, func() (bool, error) {
+		pods, err := h.KubeClient.CoreV1().Pods(ns).List(metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		if len(pods.Items) == 0 {
+			Logf("No pods found in namespace %s - checking again...")
+			return false, nil
+		}
+
+		var errs []string
+		for _, p := range pods.Items {
+			c := GetPodReadyCondition(p.Status)
+			if c == nil {
+				errs = append(errs, fmt.Sprintf("Pod %q not ready (no Ready condition)", p.Name))
+				continue
+			}
+			// This pod does not have the ready condition set to True
+			if c.Status != corev1.ConditionTrue {
+				errs = append(errs, fmt.Sprintf("Pod %q not ready: %s", p.Name, c.String()))
+			}
+		}
+
+		if len(errs) > 0 {
+			for _, err := range errs {
+				Logf(err)
+			}
+			return false, nil
+		}
+
+		return true, nil
+	})
+}
+
+// IsPodReady returns true if a pod is ready; false otherwise.
+func IsPodReady(pod *corev1.Pod) bool {
+	return IsPodReadyConditionTrue(pod.Status)
+}
+
+// IsPodReadyConditionTrue returns true if a pod is ready; false otherwise.
+func IsPodReadyConditionTrue(status corev1.PodStatus) bool {
+	condition := GetPodReadyCondition(status)
+	return condition != nil && condition.Status == corev1.ConditionTrue
+}
+
+// GetPodReadyCondition extracts the pod ready condition from the given status and returns that.
+// Returns nil if the condition is not present.
+func GetPodReadyCondition(status corev1.PodStatus) *corev1.PodCondition {
+	_, condition := GetPodCondition(&status, corev1.PodReady)
+	return condition
+}
+
+// GetPodCondition extracts the provided condition from the given status and returns that.
+// Returns nil and -1 if the condition is not present, and the index of the located condition.
+func GetPodCondition(status *corev1.PodStatus, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) {
+	if status == nil {
+		return -1, nil
+	}
+	for i := range status.Conditions {
+		if status.Conditions[i].Type == conditionType {
+			return i, &status.Conditions[i]
+		}
+	}
+	return -1, nil
+}

test/e2e/framework/log/BUILD.bazel

@@ -0,0 +1,24 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["log.go"],
+    importpath = "github.com/jetstack/cert-manager/test/e2e/framework/log",
+    tags = ["manual"],
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/github.com/onsi/ginkgo:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/framework/log/log.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2018 The Jetstack cert-manager contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package log
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo"
+)
+
+func nowStamp() string {
+	return time.Now().Format(time.StampMilli)
+}
+
+func log(level string, format string, args ...interface{}) {
+	fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...)
+}
+
+func Logf(format string, args ...interface{}) {
+	log("INFO", format, args...)
+}

test/e2e/framework/testenv.go

@@ -30,6 +30,11 @@ import (
 
 // Defines methods that help provision test environments
 
+const (
+	// How often to poll for conditions
+	Poll = 2 * time.Second
+)
+
 // CreateKubeNamespace creates a new Kubernetes Namespace for a test.
 func (f *Framework) CreateKubeNamespace(baseName string) (*v1.Namespace, error) {
 	ns := &v1.Namespace{

test/e2e/framework/util.go

@@ -29,42 +29,27 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-)
 
-const (
-	// How often to poll for conditions
-	Poll = 2 * time.Second
-
-	// Default time to wait for operations to complete
-	defaultTimeout = 30 * time.Second
-
-	longTimeout = 5 * time.Minute
+	. "github.com/jetstack/cert-manager/test/e2e/framework/log"
 )
 
 func nowStamp() string {
 	return time.Now().Format(time.StampMilli)
 }
 
-func log(level string, format string, args ...interface{}) {
-	fmt.Fprintf(GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...)
-}
-
-func Logf(format string, args ...interface{}) {
-	log("INFO", format, args...)
-}
-
 func Failf(format string, args ...interface{}) {
 	msg := fmt.Sprintf(format, args...)
-	log("INFO", msg)
+	Logf(msg)
 	Fail(nowStamp()+": "+msg, 1)
 }
 
 func Skipf(format string, args ...interface{}) {
 	msg := fmt.Sprintf(format, args...)
-	log("INFO", msg)
+	Logf("INFO", msg)
 	Skip(nowStamp() + ": " + msg)
 }
 
+// TODO: move this function into a different package
 func RbacClusterRoleHasAccessToResource(f *Framework, clusterRole string, verb string, resource string) bool {
 	By("Creating a service account")
 	viewServiceAccount := &v1.ServiceAccount{