weiguoqiang/cert-manager
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #2958 from meyskens/int-test-improve

Browse Source 
Improve integration tests with OpenAPI + webhook validation
This commit is contained in:
jetstack-bot 2020-08-13 15:41:54 +01:00 committed by GitHub
commit 4a46e167ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 281 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
pkg/api/testing/bazel.go
View File

@ -42,10 +42,19 @@ func PathForCRD(t *testing.T, name string) string {
func CRDDirectory(t *testing.T) string {
runfiles := os.Getenv("RUNFILES_DIR")
if runfiles == "" {
t.Fatalf("integration tests can only run within 'bazel test' environment")
// BAZEL_BIN_DIR allows the developer to set a path to the bazel bin directory.
// This allows for the tests to be ran outside of Bazel, for example with Delve
// the Bazel bin directory still needs to be generated using Bazel.
bazelDir := os.Getenv("BAZEL_BIN_DIR")
if runfiles == "" && bazelDir == "" {
t.Fatalf("integration tests can only run within 'bazel test' environment or have BAZEL_BIN_DIR set")
}
var path string
if bazelDir != "" {
path = filepath.Join(bazelDir, "deploy", "crds")
} else {
path = filepath.Join(runfiles, "com_github_jetstack_cert_manager", "deploy", "crds")
}
path := filepath.Join(runfiles, "com_github_jetstack_cert_manager", "deploy", "crds")
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)

3
test/integration/certificates/metrics_controller_test.go
View File

@ -117,6 +117,9 @@ func TestMetricsController(t *testing.T) {
// Create Certificate
crt := gen.Certificate(crtName,
gen.SetCertificateIssuer(cmmeta.ObjectReference{Kind: "Issuer", Name: "test-issuer"}),
gen.SetCertificateSecretName(crtName),
gen.SetCertificateCommonName(crtName),
gen.SetCertificateNamespace(namespace),
gen.SetCertificateUID("uid-1"),
)

1
test/integration/conversion/BUILD.bazel
View File

@ -9,6 +9,7 @@ go_test(
"//pkg/apis/certmanager/v1alpha3:go_default_library",
"//pkg/apis/certmanager/v1beta1:go_default_library",
"//pkg/apis/meta/v1:go_default_library",
"//pkg/util/pki:go_default_library",
"//test/integration/framework:go_default_library",
"@io_k8s_apimachinery//pkg/api/equality:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",

42
test/integration/conversion/conversion_test.go
View File

@ -18,6 +18,11 @@ package conversion
import (
"context"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/asn1"
"encoding/pem"
"testing"
"k8s.io/apimachinery/pkg/api/equality"
@ -32,10 +37,35 @@ import (
"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha3"
"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1beta1"
cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
"github.com/jetstack/cert-manager/pkg/util/pki"
"github.com/jetstack/cert-manager/test/integration/framework"
)
func generateCSR(t *testing.T) []byte {
skRSA, err := pki.GenerateRSAPrivateKey(2048)
if err != nil {
t.Fatal(err)
}
asn1Subj, _ := asn1.Marshal(pkix.Name{
CommonName: "test",
}.ToRDNSequence())
template := x509.CertificateRequest{
RawSubject: asn1Subj,
}
csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &template, skRSA)
if err != nil {
t.Fatal(err)
}
csr := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrBytes})
return csr
}
func TestConversion(t *testing.T) {
testCSR := generateCSR(t)
tests := map[string]struct {
input runtime.Object
targetGVK schema.GroupVersionKind
@ -49,6 +79,7 @@ func TestConversion(t *testing.T) {
},
Spec: v1alpha2.CertificateSpec{
SecretName: "something",
CommonName: "test",
IssuerRef: cmmeta.ObjectReference{
Name: "issuername",
},
@ -62,6 +93,7 @@ func TestConversion(t *testing.T) {
},
Spec: v1alpha3.CertificateSpec{
SecretName: "something",
CommonName: "test",
IssuerRef: cmmeta.ObjectReference{
Name: "issuername",
},
@ -75,9 +107,7 @@ func TestConversion(t *testing.T) {
Namespace: "default",
},
Spec: v1alpha2.CertificateRequestSpec{
// validating webhook isn't currently configured in test
// environment so this passes validation.
CSRPEM: []byte("a"),
CSRPEM: testCSR,
IssuerRef: cmmeta.ObjectReference{
Name: "issuername",
},
@ -90,9 +120,7 @@ func TestConversion(t *testing.T) {
Namespace: "default",
},
Spec: v1alpha3.CertificateRequestSpec{
// validating webhook isn't currently configured in test
// environment so this passes validation.
CSRPEM: []byte("a"),
CSRPEM: testCSR,
IssuerRef: cmmeta.ObjectReference{
Name: "issuername",
},
@ -107,6 +135,7 @@ func TestConversion(t *testing.T) {
},
Spec: v1alpha2.CertificateSpec{
SecretName: "abc",
CommonName: "test",
Organization: []string{"test"},
IssuerRef: cmmeta.ObjectReference{
Name: "issuername",
@ -121,6 +150,7 @@ func TestConversion(t *testing.T) {
},
Spec: v1beta1.CertificateSpec{
SecretName: "abc",
CommonName: "test",
Subject: &v1beta1.X509Subject{
Organizations: []string{"test"},
},

12
test/integration/ctl/ctl_renew_test.go
View File

@ -64,20 +64,32 @@ func TestCtlRenew(t *testing.T) {
crt1 := gen.Certificate(crt1Name,
gen.SetCertificateNamespace(ns1),
gen.SetCertificateIssuer(cmmeta.ObjectReference{Kind: "Issuer", Name: "test-issuer"}),
gen.SetCertificateSecretName("crt1"),
gen.SetCertificateCommonName("crt1"),
)
crt2 := gen.Certificate(crt2Name,
gen.SetCertificateNamespace(ns1),
gen.SetCertificateIssuer(cmmeta.ObjectReference{Kind: "Issuer", Name: "test-issuer"}),
gen.SetCertificateSecretName("crt2"),
gen.SetCertificateCommonName("crt2"),
gen.AddCertificateLabels(map[string]string{
"foo": "bar",
}),
)
crt3 := gen.Certificate(crt3Name,
gen.SetCertificateNamespace(ns2),
gen.SetCertificateIssuer(cmmeta.ObjectReference{Kind: "Issuer", Name: "test-issuer"}),
gen.SetCertificateSecretName("crt3"),
gen.SetCertificateCommonName("crt3"),
gen.AddCertificateLabels(map[string]string{
"foo": "bar",
}),
)
crt4 := gen.Certificate(crt4Name,
gen.SetCertificateIssuer(cmmeta.ObjectReference{Kind: "Issuer", Name: "test-issuer"}),
gen.SetCertificateSecretName("crt4"),
gen.SetCertificateCommonName("crt5"),
gen.SetCertificateNamespace(ns2),
)

54
test/integration/ctl/ctl_status_certificate_test.go
View File

@ -18,6 +18,11 @@ package ctl
import (
"context"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/asn1"
"encoding/pem"
"fmt"
"regexp"
"strings"
@ -35,11 +40,36 @@ import (
cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
"github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
"github.com/jetstack/cert-manager/pkg/util/pki"
"github.com/jetstack/cert-manager/test/integration/framework"
"github.com/jetstack/cert-manager/test/unit/gen"
)
func generateCSR(t *testing.T) []byte {
skRSA, err := pki.GenerateRSAPrivateKey(2048)
if err != nil {
t.Fatal(err)
}
asn1Subj, _ := asn1.Marshal(pkix.Name{
CommonName: "test",
}.ToRDNSequence())
template := x509.CertificateRequest{
RawSubject: asn1Subj,
}
csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &template, skRSA)
if err != nil {
t.Fatal(err)
}
csr := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrBytes})
return csr
}
func TestCtlStatusCert(t *testing.T) {
testCSR := generateCSR(t)
config, stopFn := framework.RunControlPlane(t)
defer stopFn()
@ -117,7 +147,7 @@ MA6koCR/K23HZfML8vT6lcHvQJp9XXaHRIe9NX/M/2f6VpfO7JjKWLou5k5a
NotAfter: &metav1.Time{Time: certIsValidTime}, Revision: &revision1},
inputArgs: []string{crt1Name},
inputNamespace: ns1,
clusterIssuer: gen.ClusterIssuer("letsencrypt-prod"),
clusterIssuer: gen.ClusterIssuer("letsencrypt-prod", gen.SetIssuerSelfSigned(cmapi.SelfSignedIssuer{})),
expErr: false,
expOutput: `^Name: testcrt-1
Namespace: testns-1
@ -151,17 +181,27 @@ No CertificateRequest found for this Certificate$`,
req: gen.CertificateRequest(req1Name,
gen.SetCertificateRequestNamespace(ns1),
gen.SetCertificateRequestAnnotations(map[string]string{cmapi.CertificateRequestRevisionAnnotationKey: fmt.Sprintf("%d", revision2)}),
gen.SetCertificateRequestCSR([]byte("dummyCSR"))),
gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{Name: "letsencrypt-prod", Kind: "Issuer"}),
gen.SetCertificateRequestCSR(testCSR)),
reqStatus: &cmapi.CertificateRequestStatus{Conditions: []cmapi.CertificateRequestCondition{reqNotReadyCond}},
issuer: gen.Issuer("letsencrypt-prod",
gen.SetIssuerNamespace(ns1),
gen.SetIssuerACME(cmacme.ACMEIssuer{})),
gen.SetIssuerACME(cmacme.ACMEIssuer{
Server: "https://dummy.acme.local/",
PrivateKey: cmmeta.SecretKeySelector{
LocalObjectReference: cmmeta.LocalObjectReference{
Name: "test",
},
Key: "test",
},
})),
secret: gen.Secret("existing-tls-secret",
gen.SetSecretNamespace(ns1),
gen.SetSecretData(map[string][]byte{"tls.crt": tlsCrt})),
order: gen.Order("example-order",
gen.SetOrderNamespace(ns1),
gen.SetOrderCsr([]byte("dummyCSR")),
gen.SetOrderCsr(testCSR),
gen.SetOrderIssuer(cmmeta.ObjectReference{Name: "letsencrypt-prod", Kind: "Issuer"}),
gen.SetOrderDNSNames("www.example.com")),
expErr: false,
expOutput: `^Name: testcrt-2
@ -217,7 +257,8 @@ Order:
req: gen.CertificateRequest(req2Name,
gen.SetCertificateRequestNamespace(ns1),
gen.SetCertificateRequestAnnotations(map[string]string{cmapi.CertificateRequestRevisionAnnotationKey: fmt.Sprintf("%d", revision2)}),
gen.SetCertificateRequestCSR([]byte("dummyCSR"))),
gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{Name: "non-existing-issuer", Kind: "Issuer"}),
gen.SetCertificateRequestCSR(testCSR)),
reqStatus: &cmapi.CertificateRequestStatus{Conditions: []cmapi.CertificateRequestCondition{reqNotReadyCond}},
issuer: nil,
expErr: false,
@ -254,8 +295,9 @@ CertificateRequest:
inputNamespace: ns1,
req: gen.CertificateRequest(req3Name,
gen.SetCertificateRequestNamespace(ns1),
gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{Name: "non-existing-clusterissuer", Kind: "ClusterIssuer"}),
gen.SetCertificateRequestAnnotations(map[string]string{cmapi.CertificateRequestRevisionAnnotationKey: fmt.Sprintf("%d", revision2)}),
gen.SetCertificateRequestCSR([]byte("dummyCSR"))),
gen.SetCertificateRequestCSR(testCSR)),
reqStatus: &cmapi.CertificateRequestStatus{Conditions: []cmapi.CertificateRequestCondition{reqNotReadyCond}},
issuer: nil,
expErr: false,

7
test/integration/framework/BUILD.bazel
View File

@ -17,24 +17,31 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//cmd/webhook/app/testing:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/testing:go_default_library",
"//pkg/client/clientset/versioned:go_default_library",
"//pkg/client/informers/externalversions:go_default_library",
"//pkg/controller:go_default_library",
"@io_k8s_api//admissionregistration/v1beta1:go_default_library",
"@io_k8s_api//core/v1:go_default_library",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions:go_default_library",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/install:go_default_library",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1beta1:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
"@io_k8s_apimachinery//pkg/runtime:go_default_library",
"@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
"@io_k8s_apimachinery//pkg/runtime/serializer/json:go_default_library",
"@io_k8s_apimachinery//pkg/runtime/serializer/versioning:go_default_library",
"@io_k8s_apimachinery//pkg/util/runtime:go_default_library",
"@io_k8s_apimachinery//pkg/util/wait:go_default_library",
"@io_k8s_client_go//discovery:go_default_library",
"@io_k8s_client_go//informers:go_default_library",
"@io_k8s_client_go//kubernetes:go_default_library",
"@io_k8s_client_go//kubernetes/scheme:go_default_library",
"@io_k8s_client_go//rest:go_default_library",
"@io_k8s_client_go//tools/record:go_default_library",
"@io_k8s_kubectl//pkg/util/openapi:go_default_library",
"@io_k8s_sigs_controller_runtime//pkg/client:go_default_library",
"@io_k8s_sigs_controller_runtime//pkg/envtest:go_default_library",
],
)

100
test/integration/framework/apiserver.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package framework
import (
"context"
"fmt"
"io/ioutil"
"os"
@ -24,6 +25,7 @@ import (
"strings"
"testing"
admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
apiextensionsinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
@ -33,9 +35,11 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer/versioning"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
webhooktesting "github.com/jetstack/cert-manager/cmd/webhook/app/testing"
"github.com/jetstack/cert-manager/pkg/api"
apitesting "github.com/jetstack/cert-manager/pkg/api/testing"
)
@ -56,16 +60,34 @@ func RunControlPlane(t *testing.T) (*rest.Config, StopFunc) {
t.Logf("Found CRD with name %q", crd.Name)
}
patchCRDConversion(crds, webhookOpts.URL, webhookOpts.CAPEM)
// environment variables
env := &envtest.Environment{
AttachControlPlaneOutput: false,
CRDs: crdsToRuntimeObjects(crds),
}
config, err := env.Start()
if err != nil {
t.Fatalf("failed to start control plane: %v", err)
}
// TODO: configure Validating and Mutating webhook
cl, err := client.New(config, client.Options{Scheme: api.Scheme})
if err != nil {
t.Fatal(err)
}
// installing the validating webhooks, not using WebhookInstallOptions as it patches the CA to be it's own
err = cl.Create(context.Background(), getValidatingWebhookConfig(webhookOpts.URL, webhookOpts.CAPEM))
if err != nil {
t.Fatal(err)
}
// installing the mutating webhooks, not using WebhookInstallOptions as it patches the CA to be it's own
err = cl.Create(context.Background(), getMutatingWebhookConfig(webhookOpts.URL, webhookOpts.CAPEM))
if err != nil {
t.Fatal(err)
}
return config, func() {
defer stopWebhook()
if err := env.Stop(); err != nil {
@ -172,3 +194,77 @@ func crdsToRuntimeObjects(in []*v1beta1.CustomResourceDefinition) []runtime.Obje
return out
}
func getValidatingWebhookConfig(url string, caPEM []byte) runtime.Object {
failurePolicy := admissionregistrationv1beta1.Fail
sideEffects := admissionregistrationv1beta1.SideEffectClassNone
validateURL := fmt.Sprintf("%s/validate", url)
webhook := admissionregistrationv1beta1.ValidatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "cert-manager-webhook",
},
Webhooks: []admissionregistrationv1beta1.ValidatingWebhook{
{
Name: "webhook.cert-manager.io",
ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
URL: &validateURL,
CABundle: caPEM,
},
Rules: []admissionregistrationv1beta1.RuleWithOperations{
{
Operations: []admissionregistrationv1beta1.OperationType{
admissionregistrationv1beta1.Create,
admissionregistrationv1beta1.Update,
},
Rule: admissionregistrationv1beta1.Rule{
APIGroups: []string{"cert-manager.io", "acme.cert-manager.io"},
APIVersions: []string{"*"},
Resources: []string{"*/*"},
},
},
},
FailurePolicy: &failurePolicy,
SideEffects: &sideEffects,
},
},
}
return &webhook
}
func getMutatingWebhookConfig(url string, caPEM []byte) runtime.Object {
failurePolicy := admissionregistrationv1beta1.Fail
sideEffects := admissionregistrationv1beta1.SideEffectClassNone
validateURL := fmt.Sprintf("%s/mutate", url)
webhook := admissionregistrationv1beta1.MutatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "cert-manager-webhook",
},
Webhooks: []admissionregistrationv1beta1.MutatingWebhook{
{
Name: "webhook.cert-manager.io",
ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
URL: &validateURL,
CABundle: caPEM,
},
Rules: []admissionregistrationv1beta1.RuleWithOperations{
{
Operations: []admissionregistrationv1beta1.OperationType{
admissionregistrationv1beta1.Create,
admissionregistrationv1beta1.Update,
},
Rule: admissionregistrationv1beta1.Rule{
APIGroups: []string{"cert-manager.io", "acme.cert-manager.io"},
APIVersions: []string{"*"},
Resources: []string{"*/*"},
},
},
},
FailurePolicy: &failurePolicy,
SideEffects: &sideEffects,
},
},
}
return &webhook
}

31
test/integration/framework/helpers.go
View File

@ -18,13 +18,18 @@ package framework
import (
"testing"
"time"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/discovery"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
"k8s.io/kubectl/pkg/util/openapi"
cmclient "github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
cminformers "github.com/jetstack/cert-manager/pkg/client/informers/externalversions"
@ -64,3 +69,29 @@ func StartInformersAndController(t *testing.T, factory informers.SharedInformerF
close(stopCh)
}
}
func WaitForOpenAPIResourcesToBeLoaded(t *testing.T, config *rest.Config, gvk schema.GroupVersionKind) {
dc, err := discovery.NewDiscoveryClientForConfig(config)
if err != nil {
t.Fatal(err)
}
err = wait.PollImmediate(time.Second, 2*time.Minute,
func() (bool, error) {
og := openapi.NewOpenAPIGetter(dc)
oapiResource, err := og.Get()
if err != nil {
return false, err
}
if oapiResource.LookupResource(gvk) != nil {
return true, nil
}
return false, nil
},
)
if err != nil {
t.Fatal("Our GVK isn't loaded into the OpenAPI resources API after waiting for 2 minutes", err)
}
}

33
tools/setup-integration-test-dependencies.sh Executable file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Copyright 2020 The Jetstack cert-manager contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
SCRIPT_ROOT=$(realpath $(dirname "${BASH_SOURCE}"))
REPO_ROOT=$(dirname "${SCRIPT_ROOT}}")
bazel build //deploy/crds:crds.regular.yaml
bazel build //hack/bin:com_coreos_etcd
bazel build //hack/bin:io_kubernetes_kube-apiserver
bazel build //hack/bin:kubectl
echo "Integration test environment is set up, do not forget to set the following environment variables:"
echo export TEST_ASSET_ETCD=${REPO_ROOT}/bazel-bin/hack/bin/etcd
echo export TEST_ASSET_KUBE_APISERVER=${REPO_ROOT}/bazel-bin/hack/bin/kube-apiserver
echo export TEST_ASSET_KUBECTL=${REPO_ROOT}/bazel-bin/hack/bin/kubectl
echo export BAZEL_BIN_DIR=${REPO_ROOT}/bazel-bin/
exec "$@"