From f2e57a1c95dbebad2dbacf0fbcdf4f51cbbdf277 Mon Sep 17 00:00:00 2001 From: Michael Tsang Date: Fri, 17 May 2019 13:22:09 +0100 Subject: [PATCH] Add ControllerSyncCallCount metric Signed-off-by: Michael Tsang --- pkg/controller/acmechallenges/BUILD.bazel | 1 + pkg/controller/acmechallenges/controller.go | 3 +++ pkg/controller/acmechallenges/sync.go | 2 ++ pkg/controller/acmeorders/BUILD.bazel | 1 + pkg/controller/acmeorders/controller.go | 3 +++ pkg/controller/acmeorders/sync.go | 2 ++ pkg/controller/certificates/sync.go | 2 ++ pkg/controller/clusterissuers/BUILD.bazel | 1 + pkg/controller/clusterissuers/controller.go | 3 +++ pkg/controller/clusterissuers/sync.go | 4 +++- pkg/controller/ingress-shim/BUILD.bazel | 1 + pkg/controller/ingress-shim/controller.go | 3 +++ pkg/controller/ingress-shim/sync.go | 2 ++ pkg/controller/ingress-shim/sync_test.go | 4 +++- pkg/controller/issuers/BUILD.bazel | 1 + pkg/controller/issuers/controller.go | 3 +++ pkg/controller/issuers/sync.go | 4 +++- pkg/metrics/metrics.go | 12 ++++++++++++ 18 files changed, 49 insertions(+), 3 deletions(-) diff --git a/pkg/controller/acmechallenges/BUILD.bazel b/pkg/controller/acmechallenges/BUILD.bazel index 6ef946075..3a795ac46 100644 --- a/pkg/controller/acmechallenges/BUILD.bazel +++ b/pkg/controller/acmechallenges/BUILD.bazel @@ -22,6 +22,7 @@ go_library( "//pkg/issuer/acme/dns/util:go_default_library", "//pkg/issuer/acme/http:go_default_library", "//pkg/logs:go_default_library", + "//pkg/metrics:go_default_library", "//third_party/crypto/acme:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/pkg/controller/acmechallenges/controller.go b/pkg/controller/acmechallenges/controller.go index c3bd3357d..efc6b5597 100644 --- a/pkg/controller/acmechallenges/controller.go +++ b/pkg/controller/acmechallenges/controller.go @@ -37,6 +37,7 @@ import ( "github.com/jetstack/cert-manager/pkg/issuer/acme/dns" "github.com/jetstack/cert-manager/pkg/issuer/acme/http" logf "github.com/jetstack/cert-manager/pkg/logs" + "github.com/jetstack/cert-manager/pkg/metrics" ) type Controller struct { @@ -63,6 +64,7 @@ type Controller struct { watchedInformers []cache.InformerSynced queue workqueue.RateLimitingInterface + metrics *metrics.Metrics scheduler *scheduler.Scheduler } @@ -104,6 +106,7 @@ func New(ctx *controllerpkg.Context) (*Controller, error) { ctrl.helper = issuer.NewHelper(ctrl.issuerLister, ctrl.clusterIssuerLister) ctrl.acmeHelper = acme.NewHelper(ctrl.secretLister, ctrl.Context.ClusterResourceNamespace) + ctrl.metrics = metrics.Default ctrl.httpSolver = http.NewSolver(ctx) var err error diff --git a/pkg/controller/acmechallenges/sync.go b/pkg/controller/acmechallenges/sync.go index c6cb7f407..c7cddef86 100644 --- a/pkg/controller/acmechallenges/sync.go +++ b/pkg/controller/acmechallenges/sync.go @@ -56,6 +56,8 @@ type solver interface { // Sync will process this ACME Challenge. // It is the core control function for ACME challenges. func (c *Controller) Sync(ctx context.Context, ch *cmapi.Challenge) (err error) { + c.metrics.ControllerSyncCallCount.WithLabelValues("acmechallenges").Inc() + log := logf.FromContext(ctx).WithValues("dnsName", ch.Spec.DNSName, "type", ch.Spec.Type) ctx = logf.NewContext(ctx, log) oldChal := ch diff --git a/pkg/controller/acmeorders/BUILD.bazel b/pkg/controller/acmeorders/BUILD.bazel index 5b2aececc..190bd3419 100644 --- a/pkg/controller/acmeorders/BUILD.bazel +++ b/pkg/controller/acmeorders/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//pkg/controller:go_default_library", "//pkg/issuer:go_default_library", "//pkg/logs:go_default_library", + "//pkg/metrics:go_default_library", "//third_party/crypto/acme:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/pkg/controller/acmeorders/controller.go b/pkg/controller/acmeorders/controller.go index 7b41955da..3f144e4d4 100644 --- a/pkg/controller/acmeorders/controller.go +++ b/pkg/controller/acmeorders/controller.go @@ -36,6 +36,7 @@ import ( controllerpkg "github.com/jetstack/cert-manager/pkg/controller" "github.com/jetstack/cert-manager/pkg/issuer" logf "github.com/jetstack/cert-manager/pkg/logs" + "github.com/jetstack/cert-manager/pkg/metrics" ) type Controller struct { @@ -57,6 +58,7 @@ type Controller struct { watchedInformers []cache.InformerSynced queue workqueue.RateLimitingInterface + metrics *metrics.Metrics // used for testing clock clock.Clock @@ -94,6 +96,7 @@ func New(ctx *controllerpkg.Context) *Controller { secretInformer := ctrl.KubeSharedInformerFactory.Core().V1().Secrets() ctrl.watchedInformers = append(ctrl.watchedInformers, secretInformer.Informer().HasSynced) ctrl.secretLister = secretInformer.Lister() + ctrl.metrics = metrics.Default ctrl.helper = issuer.NewHelper(ctrl.issuerLister, ctrl.clusterIssuerLister) ctrl.acmeHelper = acme.NewHelper(ctrl.secretLister, ctrl.Context.ClusterResourceNamespace) diff --git a/pkg/controller/acmeorders/sync.go b/pkg/controller/acmeorders/sync.go index 086834566..13ca2c059 100644 --- a/pkg/controller/acmeorders/sync.go +++ b/pkg/controller/acmeorders/sync.go @@ -48,6 +48,8 @@ var ( // - create a Challenge resource in order to fulfill required validations // - waiting for Challenge resources to enter the 'ready' state func (c *Controller) Sync(ctx context.Context, o *cmapi.Order) (err error) { + c.metrics.ControllerSyncCallCount.WithLabelValues("acmeorders").Inc() + oldOrder := o o = o.DeepCopy() diff --git a/pkg/controller/certificates/sync.go b/pkg/controller/certificates/sync.go index a597e6e6d..aed78aef9 100644 --- a/pkg/controller/certificates/sync.go +++ b/pkg/controller/certificates/sync.go @@ -69,6 +69,8 @@ var ( ) func (c *Controller) Sync(ctx context.Context, crt *v1alpha1.Certificate) (err error) { + c.metrics.ControllerSyncCallCount.WithLabelValues("certificates").Inc() + log := logf.FromContext(ctx) dbg := log.V(logf.DebugLevel) diff --git a/pkg/controller/clusterissuers/BUILD.bazel b/pkg/controller/clusterissuers/BUILD.bazel index 6e27cc66b..e4eb49918 100644 --- a/pkg/controller/clusterissuers/BUILD.bazel +++ b/pkg/controller/clusterissuers/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//pkg/controller:go_default_library", "//pkg/issuer:go_default_library", "//pkg/logs:go_default_library", + "//pkg/metrics:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", diff --git a/pkg/controller/clusterissuers/controller.go b/pkg/controller/clusterissuers/controller.go index d2ebd8326..114775548 100644 --- a/pkg/controller/clusterissuers/controller.go +++ b/pkg/controller/clusterissuers/controller.go @@ -33,6 +33,7 @@ import ( controllerpkg "github.com/jetstack/cert-manager/pkg/controller" "github.com/jetstack/cert-manager/pkg/issuer" logf "github.com/jetstack/cert-manager/pkg/logs" + "github.com/jetstack/cert-manager/pkg/metrics" ) type Controller struct { @@ -48,6 +49,7 @@ type Controller struct { watchedInformers []cache.InformerSynced queue workqueue.RateLimitingInterface + metrics *metrics.Metrics } func New(ctx *controllerpkg.Context) *Controller { @@ -66,6 +68,7 @@ func New(ctx *controllerpkg.Context) *Controller { ctrl.secretLister = secretsInformer.Lister() ctrl.issuerFactory = issuer.NewIssuerFactory(ctx) ctrl.ctx = logf.NewContext(ctx.RootContext, nil, ControllerName) + ctrl.metrics = metrics.Default return ctrl } diff --git a/pkg/controller/clusterissuers/sync.go b/pkg/controller/clusterissuers/sync.go index ba0e014d8..1b293878d 100644 --- a/pkg/controller/clusterissuers/sync.go +++ b/pkg/controller/clusterissuers/sync.go @@ -21,7 +21,7 @@ import ( "fmt" "reflect" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/errors" apiutil "github.com/jetstack/cert-manager/pkg/api/util" @@ -38,6 +38,8 @@ const ( ) func (c *Controller) Sync(ctx context.Context, iss *v1alpha1.ClusterIssuer) (err error) { + c.metrics.ControllerSyncCallCount.WithLabelValues("clusterissuers").Inc() + log := logf.FromContext(ctx) issuerCopy := iss.DeepCopy() diff --git a/pkg/controller/ingress-shim/BUILD.bazel b/pkg/controller/ingress-shim/BUILD.bazel index 0bcc6417d..3c1f5baf1 100644 --- a/pkg/controller/ingress-shim/BUILD.bazel +++ b/pkg/controller/ingress-shim/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//pkg/client/listers/certmanager/v1alpha1:go_default_library", "//pkg/controller:go_default_library", "//pkg/issuer:go_default_library", + "//pkg/metrics:go_default_library", "//pkg/util:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", diff --git a/pkg/controller/ingress-shim/controller.go b/pkg/controller/ingress-shim/controller.go index 424afd9e6..bb16f29bc 100644 --- a/pkg/controller/ingress-shim/controller.go +++ b/pkg/controller/ingress-shim/controller.go @@ -38,6 +38,7 @@ import ( cmlisters "github.com/jetstack/cert-manager/pkg/client/listers/certmanager/v1alpha1" controllerpkg "github.com/jetstack/cert-manager/pkg/controller" "github.com/jetstack/cert-manager/pkg/issuer" + "github.com/jetstack/cert-manager/pkg/metrics" "github.com/jetstack/cert-manager/pkg/util" extinformers "k8s.io/client-go/informers/extensions/v1beta1" ) @@ -72,6 +73,7 @@ type Controller struct { workerWg sync.WaitGroup syncedFuncs []cache.InformerSynced defaults defaults + metrics *metrics.Metrics } // New returns a new Certificates controller. It sets up the informer handler @@ -107,6 +109,7 @@ func New( } ctrl.helper = issuer.NewHelper(ctrl.issuerLister, ctrl.clusterIssuerLister) + ctrl.metrics = metrics.Default return ctrl } diff --git a/pkg/controller/ingress-shim/sync.go b/pkg/controller/ingress-shim/sync.go index 6f4aa4557..163d59eb6 100644 --- a/pkg/controller/ingress-shim/sync.go +++ b/pkg/controller/ingress-shim/sync.go @@ -60,6 +60,8 @@ const ( var ingressGVK = extv1beta1.SchemeGroupVersion.WithKind("Ingress") func (c *Controller) Sync(ctx context.Context, ing *extv1beta1.Ingress) error { + c.metrics.ControllerSyncCallCount.WithLabelValues("ingress-shim").Inc() + if !shouldSync(ing, c.defaults.autoCertificateAnnotations) { klog.Infof("Not syncing ingress %s/%s as it does not contain necessary annotations", ing.Namespace, ing.Name) return nil diff --git a/pkg/controller/ingress-shim/sync_test.go b/pkg/controller/ingress-shim/sync_test.go index 0f0211bc7..139f9e555 100644 --- a/pkg/controller/ingress-shim/sync_test.go +++ b/pkg/controller/ingress-shim/sync_test.go @@ -28,6 +28,7 @@ import ( "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha1" testpkg "github.com/jetstack/cert-manager/pkg/controller/test" + "github.com/jetstack/cert-manager/pkg/metrics" "github.com/jetstack/cert-manager/test/unit/gen" ) @@ -910,7 +911,8 @@ func TestSync(t *testing.T) { issuerKind: test.DefaultIssuerKind, autoCertificateAnnotations: []string{testAcmeTLSAnnotation}, }, - helper: &fakeHelper{issuer: test.Issuer}, + helper: &fakeHelper{issuer: test.Issuer}, + metrics: metrics.Default, } b.Sync() diff --git a/pkg/controller/issuers/BUILD.bazel b/pkg/controller/issuers/BUILD.bazel index fb7272cf8..9b208e93f 100644 --- a/pkg/controller/issuers/BUILD.bazel +++ b/pkg/controller/issuers/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//pkg/controller:go_default_library", "//pkg/issuer:go_default_library", "//pkg/logs:go_default_library", + "//pkg/metrics:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", diff --git a/pkg/controller/issuers/controller.go b/pkg/controller/issuers/controller.go index f27db5e3e..8ebab97f6 100644 --- a/pkg/controller/issuers/controller.go +++ b/pkg/controller/issuers/controller.go @@ -33,6 +33,7 @@ import ( controllerpkg "github.com/jetstack/cert-manager/pkg/controller" "github.com/jetstack/cert-manager/pkg/issuer" logf "github.com/jetstack/cert-manager/pkg/logs" + "github.com/jetstack/cert-manager/pkg/metrics" ) type Controller struct { @@ -49,6 +50,7 @@ type Controller struct { watchedInformers []cache.InformerSynced queue workqueue.RateLimitingInterface + metrics *metrics.Metrics } func New(ctx *controllerpkg.Context) *Controller { @@ -70,6 +72,7 @@ func New(ctx *controllerpkg.Context) *Controller { ctrl.secretLister = secretsInformer.Lister() ctrl.issuerFactory = issuer.NewIssuerFactory(ctx) ctrl.ctx = logf.NewContext(ctx.RootContext, nil, ControllerName) + ctrl.metrics = metrics.Default return ctrl } diff --git a/pkg/controller/issuers/sync.go b/pkg/controller/issuers/sync.go index 266cc5461..13e80525b 100644 --- a/pkg/controller/issuers/sync.go +++ b/pkg/controller/issuers/sync.go @@ -21,7 +21,7 @@ import ( "fmt" "reflect" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/errors" apiutil "github.com/jetstack/cert-manager/pkg/api/util" @@ -38,6 +38,8 @@ const ( ) func (c *Controller) Sync(ctx context.Context, iss *v1alpha1.Issuer) (err error) { + c.metrics.ControllerSyncCallCount.WithLabelValues("issuers").Inc() + log := logf.FromContext(ctx) issuerCopy := iss.DeepCopy() diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index d585efb4c..37feba3dc 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -84,6 +84,15 @@ var ACMEClientRequestDurationSeconds = prometheus.NewSummaryVec( []string{"scheme", "host", "path", "method", "status"}, ) +var ControllerSyncCallCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "controller_sync_call_count", + Help: "The number of sync() calls made by a controller.", + }, + []string{"controller"}, +) + type Metrics struct { ctx context.Context http.Server @@ -93,6 +102,7 @@ type Metrics struct { CertificateExpiryTimeSeconds *prometheus.GaugeVec ACMEClientRequestDurationSeconds *prometheus.SummaryVec ACMEClientRequestCount *prometheus.CounterVec + ControllerSyncCallCount *prometheus.CounterVec } func New(ctx context.Context) *Metrics { @@ -112,6 +122,7 @@ func New(ctx context.Context) *Metrics { CertificateExpiryTimeSeconds: CertificateExpiryTimeSeconds, ACMEClientRequestDurationSeconds: ACMEClientRequestDurationSeconds, ACMEClientRequestCount: ACMEClientRequestCount, + ControllerSyncCallCount: ControllerSyncCallCount, } router.Handle("/metrics", promhttp.HandlerFor(s.registry, promhttp.HandlerOpts{})) @@ -141,6 +152,7 @@ func (m *Metrics) Start(stopCh <-chan struct{}) { m.registry.MustRegister(m.CertificateExpiryTimeSeconds) m.registry.MustRegister(m.ACMEClientRequestDurationSeconds) m.registry.MustRegister(m.ACMEClientRequestCount) + m.registry.MustRegister(m.ControllerSyncCallCount) go func() { log := log.WithValues("address", m.Addr)