From a40fdd64b540f1597637748e2e7572d730fd4c4a Mon Sep 17 00:00:00 2001 From: Ashley Davis Date: Mon, 20 Jun 2022 12:19:13 +0100 Subject: [PATCH] Incease issuer and clusterissuer controller timeouts This follows ideas presented in https://github.com/cert-manager/cert-manager/pull/5214 It might be nice to add these big timeouts globally to all controllers but we're intentionally keeping these changes small and targeted for now in order to minimise the risk when backporting these changes. Signed-off-by: Ashley Davis --- pkg/controller/BUILD.bazel | 1 + .../issuing/issuing_controller.go | 3 ++- pkg/controller/clusterissuers/BUILD.bazel | 1 + pkg/controller/clusterissuers/sync.go | 5 ++-- pkg/controller/globals/BUILD.bazel | 22 +++++++++++++++ pkg/controller/globals/timeout.go | 27 +++++++++++++++++++ pkg/controller/issuers/BUILD.bazel | 1 + pkg/controller/issuers/sync.go | 5 ++-- 8 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 pkg/controller/globals/BUILD.bazel create mode 100644 pkg/controller/globals/timeout.go diff --git a/pkg/controller/BUILD.bazel b/pkg/controller/BUILD.bazel index eec331c26..edaef663a 100644 --- a/pkg/controller/BUILD.bazel +++ b/pkg/controller/BUILD.bazel @@ -70,6 +70,7 @@ filegroup( "//pkg/controller/certificates:all-srcs", "//pkg/controller/certificatesigningrequests:all-srcs", "//pkg/controller/clusterissuers:all-srcs", + "//pkg/controller/globals:all-srcs", "//pkg/controller/issuers:all-srcs", "//pkg/controller/test:all-srcs", ], diff --git a/pkg/controller/certificates/issuing/issuing_controller.go b/pkg/controller/certificates/issuing/issuing_controller.go index f4af1d209..5660c0dfe 100644 --- a/pkg/controller/certificates/issuing/issuing_controller.go +++ b/pkg/controller/certificates/issuing/issuing_controller.go @@ -157,7 +157,8 @@ func NewController( } func (c *controller) ProcessItem(ctx context.Context, key string) error { - // Set context deadline for full sync in 10 seconds + // TODO: Change to globals.DefaultControllerContextTimeout as part of a wider effort to ensure we have + // failsafe timeouts in every controller ctx, cancel := context.WithTimeout(ctx, time.Second*10) defer cancel() diff --git a/pkg/controller/clusterissuers/BUILD.bazel b/pkg/controller/clusterissuers/BUILD.bazel index 9475e70cc..018fd2886 100644 --- a/pkg/controller/clusterissuers/BUILD.bazel +++ b/pkg/controller/clusterissuers/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//pkg/client/clientset/versioned:go_default_library", "//pkg/client/listers/certmanager/v1:go_default_library", "//pkg/controller:go_default_library", + "//pkg/controller/globals:go_default_library", "//pkg/issuer:go_default_library", "//pkg/logs:go_default_library", "//pkg/util/feature:go_default_library", diff --git a/pkg/controller/clusterissuers/sync.go b/pkg/controller/clusterissuers/sync.go index c3097eb78..9b09c9e40 100644 --- a/pkg/controller/clusterissuers/sync.go +++ b/pkg/controller/clusterissuers/sync.go @@ -18,7 +18,6 @@ package clusterissuers import ( "context" - "time" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -28,6 +27,7 @@ import ( "github.com/cert-manager/cert-manager/internal/controller/feature" internalissuers "github.com/cert-manager/cert-manager/internal/controller/issuers" cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + "github.com/cert-manager/cert-manager/pkg/controller/globals" logf "github.com/cert-manager/cert-manager/pkg/logs" utilfeature "github.com/cert-manager/cert-manager/pkg/util/feature" ) @@ -41,8 +41,7 @@ const ( func (c *controller) Sync(ctx context.Context, iss *cmapi.ClusterIssuer) (err error) { log := logf.FromContext(ctx) - // allow a maximum of 90s - ctx, cancel := context.WithTimeout(ctx, time.Second*90) + ctx, cancel := context.WithTimeout(ctx, globals.DefaultControllerContextTimeout) defer cancel() issuerCopy := iss.DeepCopy() diff --git a/pkg/controller/globals/BUILD.bazel b/pkg/controller/globals/BUILD.bazel new file mode 100644 index 000000000..f0d1134b0 --- /dev/null +++ b/pkg/controller/globals/BUILD.bazel @@ -0,0 +1,22 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["timeout.go"], + importpath = "github.com/cert-manager/cert-manager/pkg/controller/globals", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/controller/globals/timeout.go b/pkg/controller/globals/timeout.go new file mode 100644 index 000000000..197630521 --- /dev/null +++ b/pkg/controller/globals/timeout.go @@ -0,0 +1,27 @@ +/* +Copyright 2022 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package globals + +import "time" + +const ( + // DefaultControllerContextTimeout is the default maximum amount of time which a single synchronize action in some controllers + // may take before the sync will be cancelled by a context timeout. + // This timeout might not be respected on all controllers thanks to backwards compatibility concerns, but it's a goal to have + // all issuers have some default timeout which represents a default upper bound on the time they're permitted to take. + DefaultControllerContextTimeout = 2 * time.Minute +) diff --git a/pkg/controller/issuers/BUILD.bazel b/pkg/controller/issuers/BUILD.bazel index f3d56c873..cc5d759d8 100644 --- a/pkg/controller/issuers/BUILD.bazel +++ b/pkg/controller/issuers/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//pkg/client/clientset/versioned:go_default_library", "//pkg/client/listers/certmanager/v1:go_default_library", "//pkg/controller:go_default_library", + "//pkg/controller/globals:go_default_library", "//pkg/issuer:go_default_library", "//pkg/logs:go_default_library", "//pkg/util/feature:go_default_library", diff --git a/pkg/controller/issuers/sync.go b/pkg/controller/issuers/sync.go index 2a8fcc780..32f3a5328 100644 --- a/pkg/controller/issuers/sync.go +++ b/pkg/controller/issuers/sync.go @@ -18,7 +18,6 @@ package issuers import ( "context" - "time" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -28,6 +27,7 @@ import ( "github.com/cert-manager/cert-manager/internal/controller/feature" internalissuers "github.com/cert-manager/cert-manager/internal/controller/issuers" cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + "github.com/cert-manager/cert-manager/pkg/controller/globals" logf "github.com/cert-manager/cert-manager/pkg/logs" utilfeature "github.com/cert-manager/cert-manager/pkg/util/feature" ) @@ -41,8 +41,7 @@ const ( func (c *controller) Sync(ctx context.Context, iss *cmapi.Issuer) (err error) { log := logf.FromContext(ctx) - // allow a maximum of 90s - ctx, cancel := context.WithTimeout(ctx, time.Second*90) + ctx, cancel := context.WithTimeout(ctx, globals.DefaultControllerContextTimeout) defer cancel() issuerCopy := iss.DeepCopy()