diff --git a/cmd/ctl/main.go b/cmd/ctl/main.go index eae38ee45..2bc2ef1e4 100644 --- a/cmd/ctl/main.go +++ b/cmd/ctl/main.go @@ -33,5 +33,6 @@ func main() { if err := cmd.Execute(); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) } } diff --git a/cmd/ctl/pkg/check/api/BUILD.bazel b/cmd/ctl/pkg/check/api/BUILD.bazel index e73033f49..b23225aec 100644 --- a/cmd/ctl/pkg/check/api/BUILD.bazel +++ b/cmd/ctl/pkg/check/api/BUILD.bazel @@ -10,8 +10,10 @@ go_library( "@com_github_spf13_cobra//:go_default_library", "@io_k8s_apimachinery//pkg/util/wait:go_default_library", "@io_k8s_cli_runtime//pkg/genericclioptions:go_default_library", - "@io_k8s_client_go//rest:go_default_library", "@io_k8s_kubectl//pkg/cmd/util:go_default_library", + "@io_k8s_kubectl//pkg/scheme:go_default_library", + "@io_k8s_kubectl//pkg/util/i18n:go_default_library", + "@io_k8s_kubectl//pkg/util/templates:go_default_library", ], ) diff --git a/cmd/ctl/pkg/check/api/api.go b/cmd/ctl/pkg/check/api/api.go index 10c2a9324..67b503322 100644 --- a/cmd/ctl/pkg/check/api/api.go +++ b/cmd/ctl/pkg/check/api/api.go @@ -18,30 +18,28 @@ package api import ( "context" - "log" + "fmt" "time" - "github.com/jetstack/cert-manager/pkg/util/cmapichecker" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cli-runtime/pkg/genericclioptions" - restclient "k8s.io/client-go/rest" cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/scheme" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/jetstack/cert-manager/pkg/util/cmapichecker" ) // Options is a struct to support check api command type Options struct { - RESTConfig *restclient.Config - // APIChecker is used to check that the cert-manager CRDs have been installed on the K8S // API server and that the cert-manager webhooks are all working APIChecker cmapichecker.Interface - // If set to true, command will wait until creating resources against the api is possible - Wait bool - // Time before timeout when waiting - Timeout time.Duration + Wait time.Duration // Time between checks when waiting Interval time.Duration @@ -49,9 +47,19 @@ type Options struct { // Namespace that is used to dry-run create the certificate resource in Namespace string + // Print details regarding encountered errors + Verbose bool + genericclioptions.IOStreams } +var checkApiDesc = templates.LongDesc(i18n.T(` +This check attempts to perform a dry-run create of a cert-manager *v1alpha2* +Certificate resource in order to verify that CRDs are installed and all the +required webhooks are reachable by the K8S API server. +We use v1alpha2 API to ensure that the API server has also connected to the +cert-manager conversion webhook.`)) + // NewOptions returns initialized Options func NewOptions(ioStreams genericclioptions.IOStreams) *Options { return &Options{ @@ -65,17 +73,20 @@ func (o *Options) Complete(factory cmdutil.Factory) error { o.Namespace, _, err = factory.ToRawKubeConfigLoader().Namespace() if err != nil { - return err + return fmt.Errorf("Error: cannot get the namespace: %v", err) } - o.RESTConfig, err = factory.ToRESTConfig() + restConfig, err := factory.ToRESTConfig() if err != nil { - return err + return fmt.Errorf("Error: cannot create the REST config: %v", err) } - o.APIChecker, err = cmapichecker.New(o.RESTConfig, o.Namespace) + // We pass the scheme that is used in the RESTConfig's NegotiatedSerializer, + // this makes sure that the cmapi is also added to NegotiatedSerializer's scheme + // see: https://github.com/jetstack/cert-manager/pull/4205#discussion_r668660271 + o.APIChecker, err = cmapichecker.New(restConfig, scheme.Scheme, o.Namespace) if err != nil { - return err + return fmt.Errorf("Error: %v", err) } return nil @@ -86,14 +97,9 @@ func NewCmdCheckApi(ctx context.Context, ioStreams genericclioptions.IOStreams, o := NewOptions(ioStreams) cmd := &cobra.Command{ - Use: "api", - Short: ` - This check attempts to perform a dry-run create of a cert-manager *v1alpha2* - Certificate resource in order to verify that CRDs are installed and all the - required webhooks are reachable by the K8S API server. - We use v1alpha2 API to ensure that the API server has also connected to the - cert-manager conversion webhook. - `, + Use: "api", + Short: "This check attempts to perform a dry-run create of a cert-manager Certificate", + Long: checkApiDesc, RunE: func(cmd *cobra.Command, args []string) error { if err := o.Complete(factory); err != nil { return err @@ -103,35 +109,39 @@ func NewCmdCheckApi(ctx context.Context, ioStreams genericclioptions.IOStreams, SilenceUsage: true, SilenceErrors: true, } - cmd.Flags().BoolVar(&o.Wait, "wait", true, "If set to true, command will wait until creating resources against the api is possible") - cmd.Flags().DurationVar(&o.Timeout, "timeout", 30*time.Second, "Time before timeout when waiting, must include unit, e.g. 5s or 10m") - cmd.Flags().DurationVar(&o.Interval, "interval", 5*time.Second, "Time between checks when waiting, must include unit, e.g. 5s or 10m") + cmd.Flags().DurationVar(&o.Wait, "wait", 1*time.Minute, "Time before timeout when waiting, must include unit, e.g. 0s or 20s") + cmd.Flags().DurationVar(&o.Interval, "interval", 5*time.Second, "Time between checks when waiting, must include unit, e.g. 1m or 10m") + cmd.Flags().BoolVarP(&o.Verbose, "verbose", "v", false, "Print details regarding encountered errors") return cmd } // Run executes check api command func (o *Options) Run(ctx context.Context) error { - log.SetFlags(0) // Disable prefixing logs with timestamps. + pollContext, cancel := context.WithTimeout(ctx, o.Wait) + defer cancel() - if !o.Wait { + pollErr := wait.PollImmediateUntil(o.Interval, func() (done bool, err error) { if err := o.APIChecker.Check(ctx); err != nil { - return err - } - - log.Print("The Kubernetes Api is ready to created cert-manager resources against") - - return nil - } - - return wait.PollImmediate(o.Interval, o.Timeout, func() (done bool, err error) { - if err := o.APIChecker.Check(ctx); err != nil { - log.Printf("%v", err) + if o.Verbose { + fmt.Fprintf(o.ErrOut, "Not ready: %v (%v)\n", err, err.Cause()) + } else { + fmt.Fprintf(o.ErrOut, "Not ready: %v\n", err) + } return false, nil } - log.Print("The Kubernetes Api is ready to created cert-manager resources against") + fmt.Fprintln(o.Out, "The cert-manager API is ready") return true, nil - }) + }, pollContext.Done()) + + if pollErr != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return pollErr + } + + return nil } diff --git a/devel/addon/certmanager/install.sh b/devel/addon/certmanager/install.sh index 8745bf1d6..1a151a30f 100755 --- a/devel/addon/certmanager/install.sh +++ b/devel/addon/certmanager/install.sh @@ -31,6 +31,7 @@ SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") # Require kubectl & helm available on PATH check_tool kubectl +check_tool kubectl-cert_manager check_tool helm # Use the current timestamp as the APP_VERSION so a rolling update will be @@ -65,3 +66,5 @@ helm upgrade \ --set "extraArgs={--dns01-recursive-nameservers=${SERVICE_IP_PREFIX}.16:53,--dns01-recursive-nameservers-only=true}" \ "$RELEASE_NAME" \ "$REPO_ROOT/bazel-bin/deploy/charts/cert-manager/cert-manager.tgz" + +kubectl cert-manager check api diff --git a/pkg/util/cmapichecker/BUILD.bazel b/pkg/util/cmapichecker/BUILD.bazel index e7797e835..0b4ce65d6 100644 --- a/pkg/util/cmapichecker/BUILD.bazel +++ b/pkg/util/cmapichecker/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -29,3 +29,15 @@ filegroup( tags = ["automanaged"], visibility = ["//visibility:public"], ) + +go_test( + name = "go_default_test", + srcs = ["cmapichecker_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/apis/certmanager/v1alpha2:go_default_library", + "@io_k8s_apimachinery//pkg/runtime:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/client:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/client/fake:go_default_library", + ], +) diff --git a/pkg/util/cmapichecker/cmapichecker.go b/pkg/util/cmapichecker/cmapichecker.go index 0492caa77..4a3fde241 100644 --- a/pkg/util/cmapichecker/cmapichecker.go +++ b/pkg/util/cmapichecker/cmapichecker.go @@ -18,6 +18,7 @@ package cmapichecker import ( "context" + "regexp" errors "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,9 +34,35 @@ import ( cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1" ) +var ( + ErrAPIServerUnreachable = errors.New("unable to connect to the Kubernetes API server") + ErrCertManagerCRDsNotFound = errors.New("the cert-manager CRDs are not yet installed on the Kubernetes API server") + ErrCertManagerAPIEndpointsNotEstablished = errors.New("the cert-manager API endpoints have not yet been published by the Kubernetes API server") + ErrWebhookConnectionFailure = errors.New("the cert-manager webhook server can't be reached yet") + ErrWebhookCertificateFailure = errors.New("the client CA bundle is not yet updated to the certificate of the cert-manager webhook") + + regexErrCertManagerCRDsNotFound = regexp.MustCompile(`^error finding the scope of the object: failed to get restmapping: no matches for kind "Certificate" in group "cert-manager.io"$`) + regexErrCertManagerAPIEndpointsNotEstablished = regexp.MustCompile(`failed calling webhook "(.*)\.cert-manager\.io": Post "(.*)\/mutate(.*)": service "(.*)-webhook" not found$`) + regexErrWebhookConnectionFailure = regexp.MustCompile(`failed calling webhook "(.*)\.cert-manager\.io": Post "(.*)\/mutate(.*)": (.*): connect: connection refused$`) + regexErrWebhookCertificateFailure = regexp.MustCompile(`Post "(.*)": x509: certificate signed by unknown authority`) +) + +type ApiCheckError struct { + SimpleError error + UnderlyingError error +} + +func (e *ApiCheckError) Error() string { + return e.SimpleError.Error() +} + +func (e *ApiCheckError) Cause() error { + return e.UnderlyingError +} + // Interface is used to check that the cert-manager CRDs have been installed and are usable. type Interface interface { - Check(context.Context) error + Check(context.Context) *ApiCheckError } type cmapiChecker struct { @@ -49,8 +76,7 @@ type cmapiChecker struct { } // New returns a cert-manager API checker -func New(restcfg *rest.Config, namespace string) (Interface, error) { - scheme := runtime.NewScheme() +func New(restcfg *rest.Config, scheme *runtime.Scheme, namespace string) (Interface, error) { if err := cmapi.AddToScheme(scheme); err != nil { return nil, errors.Wrap(err, "while configuring scheme") } @@ -86,12 +112,8 @@ func (o *cmapiChecker) Client() (client.Client, error) { // required webhooks are reachable by the K8S API server. // We use v1alpha2 API to ensure that the API server has also connected to the // cert-manager conversion webhook. -func (o *cmapiChecker) Check(ctx context.Context) error { +func (o *cmapiChecker) Check(ctx context.Context) *ApiCheckError { cert := &cmapi.Certificate{ - TypeMeta: metav1.TypeMeta{ - Kind: "Certificate", - APIVersion: "cert-manager.io/v1alpha2", - }, ObjectMeta: metav1.ObjectMeta{ GenerateName: "cmapichecker-", }, @@ -103,13 +125,42 @@ func (o *cmapiChecker) Check(ctx context.Context) error { }, }, } + + // while creating client: Get "http://localhost:8080/api?timeout=32s": dial tcp 127.0.0.1:8080: connect: connection refused cl, err := o.Client() if err != nil { - return err + return &ApiCheckError{ + SimpleError: ErrAPIServerUnreachable, + UnderlyingError: err, + } } + // error finding the scope of the object: failed to get restmapping: no matches for kind "Certificate" in group "cert-manager.io" + // Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": service "cert-manager-webhook" not found + // Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": dial tcp 10.96.38.90:443: connect: connection refused + // Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": x509: certificate signed by unknown authority (possibly because of "x509: ECDSA verification failure" while trying to verify candidate authority certificate "cert-manager-webhook-ca") + // conversion webhook for cert-manager.io/v1alpha2, Kind=Certificate failed: Post "https://cert-manager-webhook.cert-manager.svc:443/convert?timeout=30s": x509: certificate signed by unknown authority if err := cl.Create(ctx, cert); err != nil { - return errors.Wrap(err, "while attempting dry-run creation of Certificate") + return &ApiCheckError{ + SimpleError: translateToSimpleError(err), + UnderlyingError: err, + } } return nil } + +func translateToSimpleError(err error) error { + s := err.Error() + + if regexErrCertManagerCRDsNotFound.MatchString(s) { + return ErrCertManagerCRDsNotFound + } else if regexErrCertManagerAPIEndpointsNotEstablished.MatchString(s) { + return ErrCertManagerAPIEndpointsNotEstablished + } else if regexErrWebhookConnectionFailure.MatchString(s) { + return ErrWebhookConnectionFailure + } else if regexErrWebhookCertificateFailure.MatchString(s) { + return ErrWebhookCertificateFailure + } + + return err +} diff --git a/pkg/util/cmapichecker/cmapichecker_test.go b/pkg/util/cmapichecker/cmapichecker_test.go new file mode 100644 index 000000000..0d381fcdb --- /dev/null +++ b/pkg/util/cmapichecker/cmapichecker_test.go @@ -0,0 +1,144 @@ +/* +Copyright 2021 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmapichecker + +import ( + "context" + "errors" + "testing" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2" +) + +type fakeErrorClient struct { + client.Client + + newError error + createError error +} + +func (cl *fakeErrorClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if cl.createError != nil { + return cl.createError + } + + return cl.Client.Create(ctx, obj, opts...) +} + +func newFakeCmapiChecker() (*fakeErrorClient, Interface, error) { + scheme := runtime.NewScheme() + if err := cmapi.AddToScheme(scheme); err != nil { + return nil, nil, err + } + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + errorClient := &fakeErrorClient{ + Client: cl, + newError: nil, + createError: nil, + } + + return errorClient, &cmapiChecker{ + clientBuilder: func() (client.Client, error) { + if errorClient.newError != nil { + return nil, errorClient.newError + } + return errorClient, nil + }, + }, nil +} + +func TestCmapiChecker(t *testing.T) { + tests := map[string]testT{ + "check API without errors": { + newError: nil, + createError: nil, + + expectedError: "", + }, + "check API server unreachable": { + newError: errors.New("while creating client: Get \"http://localhost:8080/api?timeout=32s\": dial tcp 127.0.0.1:8080: connect: connection refused"), + createError: nil, + + expectedError: ErrAPIServerUnreachable.Error(), + }, + "check API without CRDs installed": { + newError: nil, + createError: errors.New("error finding the scope of the object: failed to get restmapping: no matches for kind \"Certificate\" in group \"cert-manager.io\""), + + expectedError: ErrCertManagerCRDsNotFound.Error(), + }, + "check API with webhook service not ready": { + newError: nil, + createError: errors.New("Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": service \"cert-manager-webhook\" not found"), + + expectedError: ErrCertManagerAPIEndpointsNotEstablished.Error(), + }, + "check API with webhook pod not accepting connections": { + newError: nil, + createError: errors.New("Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": dial tcp 10.96.38.90:443: connect: connection refused"), + + expectedError: ErrWebhookConnectionFailure.Error(), + }, + "check API with webhook certificate not updated in mutation webhook resource definitions": { + newError: nil, + createError: errors.New("Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": x509: certificate signed by unknown authority (possibly because of \"x509: ECDSA verification failure\" while trying to verify candidate authority certificate \"cert-manager-webhook-ca\""), + + expectedError: ErrWebhookCertificateFailure.Error(), + }, + "check API with webhook certificate not updated in conversion webhook resource definitions": { + newError: nil, + createError: errors.New("conversion webhook for cert-manager.io/v1alpha2, Kind=Certificate failed: Post \"https://cert-manager-webhook.cert-manager.svc:443/convert?timeout=30s\": x509: certificate signed by unknown authority"), + + expectedError: ErrWebhookCertificateFailure.Error(), + }, + } + + for n, test := range tests { + t.Run(n, func(t *testing.T) { + runTest(t, test) + }) + } +} + +type testT struct { + newError error + createError error + + expectedError string +} + +func runTest(t *testing.T, test testT) { + errorClient, checker, _ := newFakeCmapiChecker() + + errorClient.newError = test.newError + errorClient.createError = test.createError + + err := checker.Check(context.TODO()) + if err != nil { + if err.Error() != test.expectedError { + t.Errorf("error differs from expected error:\n%s\n vs \n%s", err.Error(), test.expectedError) + } + } else { + if test.expectedError != "" { + t.Errorf("expected error did not occure:\n%s", test.expectedError) + } + } +}