diff --git a/test/e2e/framework/BUILD.bazel b/test/e2e/framework/BUILD.bazel index d3650540f..81000ad79 100644 --- a/test/e2e/framework/BUILD.bazel +++ b/test/e2e/framework/BUILD.bazel @@ -15,6 +15,8 @@ go_library( "//pkg/client/clientset/versioned:go_default_library", "//test/e2e/framework/addon:go_default_library", "//test/e2e/framework/config:go_default_library", + "//test/e2e/framework/helper:go_default_library", + "//test/e2e/framework/log:go_default_library", "//test/e2e/framework/util:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", @@ -44,6 +46,8 @@ filegroup( ":package-srcs", "//test/e2e/framework/addon:all-srcs", "//test/e2e/framework/config:all-srcs", + "//test/e2e/framework/helper:all-srcs", + "//test/e2e/framework/log:all-srcs", "//test/e2e/framework/util:all-srcs", ], tags = ["automanaged"], diff --git a/test/e2e/framework/addon/base/BUILD.bazel b/test/e2e/framework/addon/base/BUILD.bazel index 4a543efbd..4dc424e3b 100644 --- a/test/e2e/framework/addon/base/BUILD.bazel +++ b/test/e2e/framework/addon/base/BUILD.bazel @@ -8,6 +8,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//test/e2e/framework/config:go_default_library", + "//test/e2e/framework/helper:go_default_library", "//test/e2e/framework/util:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", ], diff --git a/test/e2e/framework/addon/base/base.go b/test/e2e/framework/addon/base/base.go index cce7f4e3a..b8db3484a 100644 --- a/test/e2e/framework/addon/base/base.go +++ b/test/e2e/framework/addon/base/base.go @@ -19,6 +19,7 @@ limitations under the License. package base import ( + "github.com/jetstack/cert-manager/test/e2e/framework/helper" "k8s.io/client-go/kubernetes" "github.com/jetstack/cert-manager/test/e2e/framework/config" @@ -42,6 +43,12 @@ type Details struct { KubeClient kubernetes.Interface } +func (d *Details) Helper() *helper.Helper { + return &helper.Helper{ + KubeClient: d.KubeClient, + } +} + func (b *Base) Setup(c *config.Config) error { kubeConfig, err := util.LoadConfig(c.KubeConfig, c.KubeContext) if err != nil { diff --git a/test/e2e/framework/addon/chart/addon.go b/test/e2e/framework/addon/chart/addon.go index 3c4ae4fb2..c73996159 100644 --- a/test/e2e/framework/addon/chart/addon.go +++ b/test/e2e/framework/addon/chart/addon.go @@ -123,6 +123,11 @@ func (c *Chart) Provision() error { return fmt.Errorf("error install helm chart: %v", err) } + err = c.Tiller.Base.Details().Helper().WaitForAllPodsRunningInNamespace(c.Namespace) + if err != nil { + return err + } + return nil } diff --git a/test/e2e/framework/addon/tiller/addon.go b/test/e2e/framework/addon/tiller/addon.go index 319113401..370effcff 100644 --- a/test/e2e/framework/addon/tiller/addon.go +++ b/test/e2e/framework/addon/tiller/addon.go @@ -18,7 +18,6 @@ package tiller import ( "fmt" - "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -254,32 +253,9 @@ func (t *Tiller) Provision() error { } // otherwise lookup the newly created pods name - kubeClient := t.Base.Details().KubeClient - retries := 10 - for { - pods, err := kubeClient.CoreV1().Pods(t.Namespace).List(metav1.ListOptions{ - LabelSelector: "name=tiller", - }) - if err != nil { - return err - } - if len(pods.Items) == 0 { - if retries == 0 { - return fmt.Errorf("failed to create tiller pod within 10s") - } - retries-- - time.Sleep(time.Second * 2) - continue - } - tillerPod := pods.Items[0] - // If the vault pod exists but is just waiting to be created, we allow - // it a bit longer. - if len(tillerPod.Status.ContainerStatuses) == 0 || !tillerPod.Status.ContainerStatuses[0].Ready { - retries-- - time.Sleep(time.Second * 5) - continue - } - break + err = t.Base.Details().Helper().WaitForAllPodsRunningInNamespace(t.Namespace) + if err != nil { + return err } return nil diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index d6fe58200..e229af56d 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -17,6 +17,7 @@ limitations under the License. package framework import ( + "github.com/jetstack/cert-manager/test/e2e/framework/helper" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/api/core/v1" @@ -157,6 +158,12 @@ func (f *Framework) RequireAddon(a addon.Addon) { }) } +func (f *Framework) Helper() *helper.Helper { + return &helper.Helper{ + KubeClient: f.KubeClientSet, + } +} + // CertManagerDescribe is a wrapper function for ginkgo describe. Adds namespacing. func CertManagerDescribe(text string, body func()) bool { return Describe("[cert-manager] "+text, body) diff --git a/test/e2e/framework/helper/BUILD.bazel b/test/e2e/framework/helper/BUILD.bazel new file mode 100644 index 000000000..a43d5f410 --- /dev/null +++ b/test/e2e/framework/helper/BUILD.bazel @@ -0,0 +1,34 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "helper.go", + "pod_start.go", + ], + importpath = "github.com/jetstack/cert-manager/test/e2e/framework/helper", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//test/e2e/framework/log:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e/framework/helper/helper.go b/test/e2e/framework/helper/helper.go new file mode 100644 index 000000000..b68b3a290 --- /dev/null +++ b/test/e2e/framework/helper/helper.go @@ -0,0 +1,24 @@ +/* +Copyright 2018 The Jetstack cert-manager contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import "k8s.io/client-go/kubernetes" + +// Helper provides methods for common operations needed during tests. +type Helper struct { + KubeClient kubernetes.Interface +} diff --git a/test/e2e/framework/helper/pod_start.go b/test/e2e/framework/helper/pod_start.go new file mode 100644 index 000000000..35ae1e140 --- /dev/null +++ b/test/e2e/framework/helper/pod_start.go @@ -0,0 +1,113 @@ +/* +Copyright 2018 The Jetstack cert-manager contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + . "github.com/jetstack/cert-manager/test/e2e/framework/log" +) + +const ( + // Poll is how often the API is polled in Wait operations by default + Poll = time.Second * 2 + + // PodStartTimeout is the default amount of time to wait in pod start operations + PodStartTimeout = time.Minute +) + +// WaitForAllPodsRunningInNamespace waits default amount of time (PodStartTimeout) +// for all pods in the specified namespace to become running. +func (h *Helper) WaitForAllPodsRunningInNamespace(ns string) error { + return h.WaitForAllPodsRunningInNamespaceTimeout(ns, PodStartTimeout) +} + +func (h *Helper) WaitForAllPodsRunningInNamespaceTimeout(ns string, timeout time.Duration) error { + By("Waiting " + timeout.String() + " for all pods in namespace '" + ns + "' to be Ready") + return wait.PollImmediate(Poll, timeout, func() (bool, error) { + pods, err := h.KubeClient.CoreV1().Pods(ns).List(metav1.ListOptions{}) + if err != nil { + return false, err + } + + if len(pods.Items) == 0 { + Logf("No pods found in namespace %s - checking again...") + return false, nil + } + + var errs []string + for _, p := range pods.Items { + c := GetPodReadyCondition(p.Status) + if c == nil { + errs = append(errs, fmt.Sprintf("Pod %q not ready (no Ready condition)", p.Name)) + continue + } + // This pod does not have the ready condition set to True + if c.Status != corev1.ConditionTrue { + errs = append(errs, fmt.Sprintf("Pod %q not ready: %s", p.Name, c.String())) + } + } + + if len(errs) > 0 { + for _, err := range errs { + Logf(err) + } + return false, nil + } + + return true, nil + }) +} + +// IsPodReady returns true if a pod is ready; false otherwise. +func IsPodReady(pod *corev1.Pod) bool { + return IsPodReadyConditionTrue(pod.Status) +} + +// IsPodReadyConditionTrue returns true if a pod is ready; false otherwise. +func IsPodReadyConditionTrue(status corev1.PodStatus) bool { + condition := GetPodReadyCondition(status) + return condition != nil && condition.Status == corev1.ConditionTrue +} + +// GetPodReadyCondition extracts the pod ready condition from the given status and returns that. +// Returns nil if the condition is not present. +func GetPodReadyCondition(status corev1.PodStatus) *corev1.PodCondition { + _, condition := GetPodCondition(&status, corev1.PodReady) + return condition +} + +// GetPodCondition extracts the provided condition from the given status and returns that. +// Returns nil and -1 if the condition is not present, and the index of the located condition. +func GetPodCondition(status *corev1.PodStatus, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) { + if status == nil { + return -1, nil + } + for i := range status.Conditions { + if status.Conditions[i].Type == conditionType { + return i, &status.Conditions[i] + } + } + return -1, nil +} diff --git a/test/e2e/framework/log/BUILD.bazel b/test/e2e/framework/log/BUILD.bazel new file mode 100644 index 000000000..703c15edd --- /dev/null +++ b/test/e2e/framework/log/BUILD.bazel @@ -0,0 +1,24 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["log.go"], + importpath = "github.com/jetstack/cert-manager/test/e2e/framework/log", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = ["//vendor/github.com/onsi/ginkgo:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e/framework/log/log.go b/test/e2e/framework/log/log.go new file mode 100644 index 000000000..31b00975e --- /dev/null +++ b/test/e2e/framework/log/log.go @@ -0,0 +1,36 @@ +/* +Copyright 2018 The Jetstack cert-manager contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package log + +import ( + "fmt" + "time" + + "github.com/onsi/ginkgo" +) + +func nowStamp() string { + return time.Now().Format(time.StampMilli) +} + +func log(level string, format string, args ...interface{}) { + fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...) +} + +func Logf(format string, args ...interface{}) { + log("INFO", format, args...) +} diff --git a/test/e2e/framework/testenv.go b/test/e2e/framework/testenv.go index 0a3bda2b0..7940a680a 100644 --- a/test/e2e/framework/testenv.go +++ b/test/e2e/framework/testenv.go @@ -30,6 +30,11 @@ import ( // Defines methods that help provision test environments +const ( + // How often to poll for conditions + Poll = 2 * time.Second +) + // CreateKubeNamespace creates a new Kubernetes Namespace for a test. func (f *Framework) CreateKubeNamespace(baseName string) (*v1.Namespace, error) { ns := &v1.Namespace{ diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index a40434a17..10f664a18 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -29,42 +29,27 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" -) -const ( - // How often to poll for conditions - Poll = 2 * time.Second - - // Default time to wait for operations to complete - defaultTimeout = 30 * time.Second - - longTimeout = 5 * time.Minute + . "github.com/jetstack/cert-manager/test/e2e/framework/log" ) func nowStamp() string { return time.Now().Format(time.StampMilli) } -func log(level string, format string, args ...interface{}) { - fmt.Fprintf(GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...) -} - -func Logf(format string, args ...interface{}) { - log("INFO", format, args...) -} - func Failf(format string, args ...interface{}) { msg := fmt.Sprintf(format, args...) - log("INFO", msg) + Logf(msg) Fail(nowStamp()+": "+msg, 1) } func Skipf(format string, args ...interface{}) { msg := fmt.Sprintf(format, args...) - log("INFO", msg) + Logf("INFO", msg) Skip(nowStamp() + ": " + msg) } +// TODO: move this function into a different package func RbacClusterRoleHasAccessToResource(f *Framework, clusterRole string, verb string, resource string) bool { By("Creating a service account") viewServiceAccount := &v1.ServiceAccount{