From 2df84b92d09919b13f0a5baf5968d06e4919d9b8 Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Mon, 26 Jul 2021 19:08:28 +0100 Subject: [PATCH] Update e2e tests to use beta Ingress APIs if v1 is unavailable Signed-off-by: Jake Sanders --- .../suite/conformance/certificates/tests.go | 101 ++++++++---- .../suite/issuers/acme/certificate/http01.go | 153 ++++++++++++------ test/e2e/util/util.go | 53 +++++- 3 files changed, 229 insertions(+), 78 deletions(-) diff --git a/test/e2e/suite/conformance/certificates/tests.go b/test/e2e/suite/conformance/certificates/tests.go index 082d64395..533f4ca61 100644 --- a/test/e2e/suite/conformance/certificates/tests.go +++ b/test/e2e/suite/conformance/certificates/tests.go @@ -18,6 +18,8 @@ package certificates import ( "context" + networkingv1 "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" "time" . "github.com/onsi/ginkgo" @@ -641,20 +643,38 @@ func (s *Suite) Define() { }, featureset.ReusePrivateKeyFeature, featureset.OnlySAN) s.it(f, "should issue a certificate for a single distinct DNS Name defined by an ingress with annotations", func(issuerRef cmmeta.ObjectReference) { - ingClient := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace.Name) + var certName string + switch { + case e2eutil.HasAPIVersion(f.KubeClientSet.Discovery(), networkingv1.SchemeGroupVersion.String()): + ingClient := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace.Name) - name := "testcert-ingress" - secretName := "testcert-ingress-tls" + name := "testcert-ingress" + secretName := "testcert-ingress-tls" - By("Creating an Ingress with the issuer name annotation set") - ingress, err := ingClient.Create(context.TODO(), e2eutil.NewIngress(name, secretName, map[string]string{ - "cert-manager.io/issuer": issuerRef.Name, - "cert-manager.io/issuer-kind": issuerRef.Kind, - "cert-manager.io/issuer-group": issuerRef.Group, - }, e2eutil.RandomSubdomain(s.DomainSuffix)), metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + By("Creating an Ingress with the issuer name annotation set") + ingress, err := ingClient.Create(context.TODO(), e2eutil.NewIngress(name, secretName, map[string]string{ + "cert-manager.io/issuer": issuerRef.Name, + "cert-manager.io/issuer-kind": issuerRef.Kind, + "cert-manager.io/issuer-group": issuerRef.Group, + }, e2eutil.RandomSubdomain(s.DomainSuffix)), metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + certName = ingress.Spec.TLS[0].SecretName + case e2eutil.HasAPIVersion(f.KubeClientSet.Discovery(), networkingv1beta1.SchemeGroupVersion.String()): + ingClient := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace.Name) + name := "testcert-ingress" + secretName := "testcert-ingress-tls" - certName := ingress.Spec.TLS[0].SecretName + By("Creating an Ingress with the issuer name annotation set") + ingress, err := ingClient.Create(context.TODO(), e2eutil.NewV1Beta1Ingress(name, secretName, map[string]string{ + "cert-manager.io/issuer": issuerRef.Name, + "cert-manager.io/issuer-kind": issuerRef.Kind, + "cert-manager.io/issuer-group": issuerRef.Group, + }, e2eutil.RandomSubdomain(s.DomainSuffix)), metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + certName = ingress.Spec.TLS[0].SecretName + default: + Fail("Neither " + networkingv1.SchemeGroupVersion.String() + " nor " + networkingv1beta1.SchemeGroupVersion.String() + " were discovered in the API server") + } By("Waiting for the Certificate to exist...") Expect(e2eutil.WaitForCertificateToExist( @@ -662,7 +682,7 @@ func (s *Suite) Define() { )).NotTo(HaveOccurred()) By("Waiting for the Certificate to be issued...") - _, err = f.Helper().WaitForCertificateReady(f.Namespace.Name, certName, time.Minute*5) + _, err := f.Helper().WaitForCertificateReady(f.Namespace.Name, certName, time.Minute*5) Expect(err).NotTo(HaveOccurred()) By("Validating the issued Certificate...") @@ -671,26 +691,51 @@ func (s *Suite) Define() { }, featureset.OnlySAN) s.it(f, "should issue a certificate defined by an ingress with certificate field annotations", func(issuerRef cmmeta.ObjectReference) { - ingClient := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace.Name) - - name := "testcert-ingress" - secretName := "testcert-ingress-tls" + var certName string domain := e2eutil.RandomSubdomain(s.DomainSuffix) duration := time.Hour * 999 renewBefore := time.Hour * 111 - By("Creating an Ingress with annotations for issuerRef and other Certificate fields") - ingress, err := ingClient.Create(context.TODO(), e2eutil.NewIngress(name, secretName, map[string]string{ - "cert-manager.io/issuer": issuerRef.Name, - "cert-manager.io/issuer-kind": issuerRef.Kind, - "cert-manager.io/issuer-group": issuerRef.Group, - "cert-manager.io/common-name": domain, - "cert-manager.io/duration": duration.String(), - "cert-manager.io/renew-before": renewBefore.String(), - }, domain), metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + switch { + case e2eutil.HasAPIVersion(f.KubeClientSet.Discovery(), networkingv1.SchemeGroupVersion.String()): + ingClient := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace.Name) - certName := ingress.Spec.TLS[0].SecretName + name := "testcert-ingress" + secretName := "testcert-ingress-tls" + + By("Creating an Ingress with annotations for issuerRef and other Certificate fields") + ingress, err := ingClient.Create(context.TODO(), e2eutil.NewIngress(name, secretName, map[string]string{ + "cert-manager.io/issuer": issuerRef.Name, + "cert-manager.io/issuer-kind": issuerRef.Kind, + "cert-manager.io/issuer-group": issuerRef.Group, + "cert-manager.io/common-name": domain, + "cert-manager.io/duration": duration.String(), + "cert-manager.io/renew-before": renewBefore.String(), + }, domain), metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + certName = ingress.Spec.TLS[0].SecretName + case e2eutil.HasAPIVersion(f.KubeClientSet.Discovery(), networkingv1beta1.SchemeGroupVersion.String()): + ingClient := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace.Name) + + name := "testcert-ingress" + secretName := "testcert-ingress-tls" + + By("Creating an Ingress with annotations for issuerRef and other Certificate fields") + ingress, err := ingClient.Create(context.TODO(), e2eutil.NewV1Beta1Ingress(name, secretName, map[string]string{ + "cert-manager.io/issuer": issuerRef.Name, + "cert-manager.io/issuer-kind": issuerRef.Kind, + "cert-manager.io/issuer-group": issuerRef.Group, + "cert-manager.io/common-name": domain, + "cert-manager.io/duration": duration.String(), + "cert-manager.io/renew-before": renewBefore.String(), + }, domain), metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + certName = ingress.Spec.TLS[0].SecretName + default: + Fail("Neither " + networkingv1.SchemeGroupVersion.String() + " nor " + networkingv1beta1.SchemeGroupVersion.String() + " were discovered in the API server") + } By("Waiting for the Certificate to exist...") Expect(e2eutil.WaitForCertificateToExist( @@ -698,7 +743,7 @@ func (s *Suite) Define() { )).NotTo(HaveOccurred()) By("Waiting for the Certificate to be issued...") - _, err = f.Helper().WaitForCertificateReady(f.Namespace.Name, certName, time.Minute*5) + _, err := f.Helper().WaitForCertificateReady(f.Namespace.Name, certName, time.Minute*5) Expect(err).NotTo(HaveOccurred()) // Verify that the ingres-shim has translated all the supplied diff --git a/test/e2e/suite/issuers/acme/certificate/http01.go b/test/e2e/suite/issuers/acme/certificate/http01.go index 79f29a8ee..f9e631e9f 100644 --- a/test/e2e/suite/issuers/acme/certificate/http01.go +++ b/test/e2e/suite/issuers/acme/certificate/http01.go @@ -21,17 +21,12 @@ import ( "crypto/tls" "crypto/x509" "fmt" + networkingv1beta1 "k8s.io/api/networking/v1beta1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" "strings" "time" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/utils/pointer" - cmacme "github.com/jetstack/cert-manager/pkg/apis/acme/v1" v1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1" cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1" @@ -43,6 +38,12 @@ import ( "github.com/jetstack/cert-manager/test/e2e/util" e2eutil "github.com/jetstack/cert-manager/test/e2e/util" "github.com/jetstack/cert-manager/test/unit/gen" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" ) const foreverTestTimeout = time.Second * 60 @@ -226,17 +227,29 @@ var _ = framework.CertManagerDescribe("ACME Certificate (HTTP01)", func() { }) It("should obtain a signed certificate with a single CN from the ACME server when putting an annotation on an ingress resource", func() { - ingClient := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace.Name) + + switch { + case util.HasAPIVersion(f.KubeClientSet.Discovery(), networkingv1.SchemeGroupVersion.String()): + ingClient := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace.Name) + By("Creating an Ingress with the issuer name annotation set") + _, err := ingClient.Create(context.TODO(), util.NewIngress(certificateSecretName, certificateSecretName, map[string]string{ + "cert-manager.io/issuer": issuerName, + }, acmeIngressDomain), metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + case util.HasAPIVersion(f.KubeClientSet.Discovery(), networkingv1beta1.SchemeGroupVersion.String()): + ingClient := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace.Name) + By("Creating an Ingress with the issuer name annotation set") + _, err := ingClient.Create(context.TODO(), util.NewV1Beta1Ingress(certificateSecretName, certificateSecretName, map[string]string{ + "cert-manager.io/issuer": issuerName, + }, acmeIngressDomain), metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + default: + Fail("Neither " + networkingv1.SchemeGroupVersion.String() + " nor " + networkingv1beta1.SchemeGroupVersion.String() + " were discovered in the API server") + } + certClient := f.CertManagerClientSet.CertmanagerV1().Certificates(f.Namespace.Name) - - By("Creating an Ingress with the issuer name annotation set") - _, err := ingClient.Create(context.TODO(), util.NewIngress(certificateSecretName, certificateSecretName, map[string]string{ - "cert-manager.io/issuer": issuerName, - }, acmeIngressDomain), metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - By("Waiting for Certificate to exist") - err = util.WaitForCertificateToExist(certClient, certificateSecretName, foreverTestTimeout) + err := util.WaitForCertificateToExist(certClient, certificateSecretName, foreverTestTimeout) Expect(err).NotTo(HaveOccurred()) By("Waiting for the Certificate to be issued...") @@ -287,36 +300,39 @@ var _ = framework.CertManagerDescribe("ACME Certificate (HTTP01)", func() { // create an ingress that points at nothing, but has the TLS redirect annotation set // using the TLS secret that we just got from the self-sign - ingress := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace.Name) - _, err = ingress.Create(context.TODO(), &networkingv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: fixedIngressName, - Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/force-ssl-redirect": "true", - "kubernetes.io/ingress.class": "nginx", - }, - }, - Spec: networkingv1.IngressSpec{ - IngressClassName: pointer.StringPtr("nginx"), - TLS: []networkingv1.IngressTLS{ - { - Hosts: []string{acmeIngressDomain}, - SecretName: secretname, + + switch { + case util.HasAPIVersion(f.KubeClientSet.Discovery(), networkingv1.SchemeGroupVersion.String()): + ingress := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace.Name) + _, err = ingress.Create(context.TODO(), &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: fixedIngressName, + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/force-ssl-redirect": "true", }, }, - Rules: []networkingv1.IngressRule{ - { - Host: acmeIngressDomain, - IngressRuleValue: networkingv1.IngressRuleValue{ - HTTP: &networkingv1.HTTPIngressRuleValue{ - Paths: []networkingv1.HTTPIngressPath{ - { - Path: "/", - Backend: networkingv1.IngressBackend{ - Service: &networkingv1.IngressServiceBackend{ - Name: "doesnotexist", - Port: networkingv1.ServiceBackendPort{ - Number: 443, + Spec: networkingv1.IngressSpec{ + IngressClassName: pointer.StringPtr("nginx"), + TLS: []networkingv1.IngressTLS{ + { + Hosts: []string{acmeIngressDomain}, + SecretName: secretname, + }, + }, + Rules: []networkingv1.IngressRule{ + { + Host: acmeIngressDomain, + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "doesnotexist", + Port: networkingv1.ServiceBackendPort{ + Number: 443, + }, }, }, }, @@ -326,11 +342,50 @@ var _ = framework.CertManagerDescribe("ACME Certificate (HTTP01)", func() { }, }, }, - }, - }, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + case util.HasAPIVersion(f.KubeClientSet.Discovery(), networkingv1beta1.SchemeGroupVersion.String()): + ingress := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace.Name) + _, err = ingress.Create(context.TODO(), &networkingv1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: fixedIngressName, + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/force-ssl-redirect": "true", + }, + }, + Spec: networkingv1beta1.IngressSpec{ + IngressClassName: pointer.StringPtr("nginx"), + TLS: []networkingv1beta1.IngressTLS{ + { + Hosts: []string{acmeIngressDomain}, + SecretName: secretname, + }, + }, + Rules: []networkingv1beta1.IngressRule{ + { + Host: acmeIngressDomain, + IngressRuleValue: networkingv1beta1.IngressRuleValue{ + HTTP: &networkingv1beta1.HTTPIngressRuleValue{ + Paths: []networkingv1beta1.HTTPIngressPath{ + { + Path: "/", + Backend: networkingv1beta1.IngressBackend{ + ServiceName: "doesnotexist", + ServicePort: intstr.FromInt(443), + }, + }, + }, + }, + }, + }, + }, + }, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + default: + Fail("Neither " + networkingv1.SchemeGroupVersion.String() + " nor " + networkingv1beta1.SchemeGroupVersion.String() + " were discovered in the API server") + } - By("Creating a Certificate") // This is a special cert for the test suite, where we specify an ingress rather than a // class By("Creating a Certificate") diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index 04cffb310..707759543 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -25,6 +25,9 @@ import ( "crypto/x509/pkix" "encoding/pem" "fmt" + networkingv1beta1 "k8s.io/api/networking/v1beta1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/discovery" "net" "net/url" "time" @@ -440,7 +443,7 @@ func NewIngress(name, secretName string, annotations map[string]string, dnsNames PathType: pathTypePrefix(), Backend: networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ - Name: "http", + Name: "somesvc", Port: networkingv1.ServiceBackendPort{ Number: 80, }, @@ -456,6 +459,41 @@ func NewIngress(name, secretName string, annotations map[string]string, dnsNames } } +func NewV1Beta1Ingress(name, secretName string, annotations map[string]string, dnsNames ...string) *networkingv1beta1.Ingress { + return &networkingv1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: annotations, + }, + Spec: networkingv1beta1.IngressSpec{ + TLS: []networkingv1beta1.IngressTLS{ + { + Hosts: dnsNames, + SecretName: secretName, + }, + }, + Rules: []networkingv1beta1.IngressRule{ + { + Host: dnsNames[0], + IngressRuleValue: networkingv1beta1.IngressRuleValue{ + HTTP: &networkingv1beta1.HTTPIngressRuleValue{ + Paths: []networkingv1beta1.HTTPIngressPath{ + { + Path: "/", + Backend: networkingv1beta1.IngressBackend{ + ServiceName: "somesvc", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + }, + } +} + func pathTypePrefix() *networkingv1.PathType { p := networkingv1.PathTypePrefix return &p @@ -535,3 +573,16 @@ func ptrPort(port int32) *gwapiv1alpha1.PortNumber { p := gwapiv1alpha1.PortNumber(port) return &p } + +// HasAPIVersion lets you know if an API exists in the discovery API +// calling this function always performs a request to the API server. +func HasAPIVersion(d discovery.DiscoveryInterface, GroupVersion string) bool { + resourceList, err := d.ServerResourcesForGroupVersion(GroupVersion) + if err != nil { + return false + } + if len(resourceList.APIResources) > 0 { + return true + } + return false +}