From 9e98d7b94882c23b89c0797f8273a432eac25db9 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Mon, 18 May 2020 17:41:14 +0100 Subject: [PATCH] Updates metrics package to be better consumable Signed-off-by: JoshVanL --- pkg/metrics/BUILD.bazel | 10 +- pkg/metrics/metrics.go | 454 +++++++++++++++--------------------- pkg/metrics/metrics_test.go | 404 ++++++++++++++++---------------- 3 files changed, 381 insertions(+), 487 deletions(-) diff --git a/pkg/metrics/BUILD.bazel b/pkg/metrics/BUILD.bazel index fea6df740..c3657cefe 100644 --- a/pkg/metrics/BUILD.bazel +++ b/pkg/metrics/BUILD.bazel @@ -8,17 +8,11 @@ go_library( deps = [ "//pkg/apis/certmanager/v1alpha2:go_default_library", "//pkg/apis/meta/v1:go_default_library", - "//pkg/client/listers/certmanager/v1alpha2:go_default_library", "//pkg/logs:go_default_library", - "//pkg/util/errors:go_default_library", - "//pkg/util/kube:go_default_library", + "@com_github_go_logr_logr//:go_default_library", "@com_github_gorilla_mux//:go_default_library", "@com_github_prometheus_client_golang//prometheus:go_default_library", "@com_github_prometheus_client_golang//prometheus/promhttp:go_default_library", - "@io_k8s_apimachinery//pkg/api/errors:go_default_library", - "@io_k8s_apimachinery//pkg/labels:go_default_library", - "@io_k8s_apimachinery//pkg/util/wait:go_default_library", - "@io_k8s_client_go//listers/core/v1:go_default_library", "@io_k8s_client_go//tools/cache:go_default_library", ], ) @@ -44,6 +38,8 @@ go_test( deps = [ "//pkg/apis/certmanager/v1alpha2:go_default_library", "//pkg/apis/meta/v1:go_default_library", + "//pkg/logs/testing:go_default_library", + "//test/unit/gen:go_default_library", "@com_github_prometheus_client_golang//prometheus/testutil:go_default_library", "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", ], diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a39ec5227..2cdeb9e99 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Jetstack cert-manager contributors. +Copyright 2020 The Jetstack cert-manager contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,370 +18,280 @@ limitations under the License. // cert-manager exposes the following metrics: // certificate_expiration_timestamp_seconds{name, namespace} // certificate_ready_status{name, namespace, condition} +// acme_client_request_count{"scheme", "host", "path", "method", "status"} +// acme_client_request_duration_seconds{"scheme", "host", "path", "method", "status"} +// controller_sync_call_count{"controller"} package metrics import ( "context" - "crypto/x509" "net/http" "sync" "time" + "github.com/go-logr/logr" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/wait" - corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" - "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2" + cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2" cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1" - cmlisters "github.com/jetstack/cert-manager/pkg/client/listers/certmanager/v1alpha2" logf "github.com/jetstack/cert-manager/pkg/logs" - "github.com/jetstack/cert-manager/pkg/util/errors" - "github.com/jetstack/cert-manager/pkg/util/kube" ) const ( // Namespace is the namespace for cert-manager metric names namespace = "certmanager" - prometheusMetricsServerAddress = "0.0.0.0:9402" + prometheusMetricsServerAddress = "127.0.0.1:9402" prometheusMetricsServerShutdownTimeout = 5 * time.Second prometheusMetricsServerReadTimeout = 8 * time.Second prometheusMetricsServerWriteTimeout = 8 * time.Second prometheusMetricsServerMaxHeaderBytes = 1 << 20 // 1 MiB ) -var readyConditionStatuses = [...]string{string(cmmeta.ConditionTrue), string(cmmeta.ConditionFalse), string(cmmeta.ConditionUnknown)} - -// Default set of metrics -var Default = New(logf.NewContext(context.Background(), logf.Log.WithName("metrics"))) - -var CertificateExpiryTimeSeconds = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "certificate_expiration_timestamp_seconds", - Help: "The date after which the certificate expires. Expressed as a Unix Epoch Time.", - }, - []string{"name", "namespace"}, -) - -var CertificateReadyStatus = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "certificate_ready_status", - Help: "The ready status of the certificate.", - }, - []string{"name", "namespace", "condition"}, -) - -// ACMEClientRequestCount is a Prometheus summary to collect the number of -// requests made to each endpoint with the ACME client. -var ACMEClientRequestCount = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "acme_client_request_count", - Help: "The number of requests made by the ACME client.", - Subsystem: "http", - }, - []string{"scheme", "host", "path", "method", "status"}, -) - -// ACMEClientRequestDurationSeconds is a Prometheus summary to collect request -// times for the ACME client. -var ACMEClientRequestDurationSeconds = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: namespace, - Name: "acme_client_request_duration_seconds", - Help: "The HTTP request latencies in seconds for the ACME client.", - Subsystem: "http", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, - []string{"scheme", "host", "path", "method", "status"}, -) - -var ControllerSyncCallCount = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "controller_sync_call_count", - Help: "The number of sync() calls made by a controller.", - }, - []string{"controller"}, -) - -// registeredCertificates holds the set of all certificates which are currently -// registered by Prometheus -var registeredCertificates = &struct { - certificates map[string]struct{} - mtx sync.Mutex -}{ - certificates: make(map[string]struct{}), -} - -// cleanUpFunctions are functions called to clean up metrics which refer to -// deleted certificates, inputs are name and namespace of the certificate -var cleanUpFunctions = []func(string, string){ - metricCleanUpCertificate(CertificateExpiryTimeSeconds), - metricCleanUpCertificateWith(CertificateReadyStatus, readyConditionStatuses[:]), -} - -type cleanableMetric interface { - DeleteLabelValues(...string) bool -} - +// Metrics is designed to be a shared object for updating the metrics exposed +// by cert-manager type Metrics struct { - ctx context.Context - http.Server - activeCertificates cmlisters.CertificateLister + log logr.Logger + registry *prometheus.Registry + server *http.Server - // TODO (@dippynark): switch this to use an interface to make it testable - registry *prometheus.Registry - CertificateExpiryTimeSeconds *prometheus.GaugeVec - CertificateReadyStatus *prometheus.GaugeVec - ACMEClientRequestDurationSeconds *prometheus.SummaryVec - ACMEClientRequestCount *prometheus.CounterVec - ControllerSyncCallCount *prometheus.CounterVec + mux sync.Mutex + + certificateExpiryTimeSeconds *prometheus.GaugeVec + certificateReadyStatus *prometheus.GaugeVec + acmeClientRequestDurationSeconds *prometheus.SummaryVec + acmeClientRequestCount *prometheus.CounterVec + controllerSyncCallCount *prometheus.CounterVec + + registeredCertificates map[string]struct{} } -func New(ctx context.Context) *Metrics { +var readyConditionStatuses = [...]cmmeta.ConditionStatus{cmmeta.ConditionTrue, cmmeta.ConditionFalse, cmmeta.ConditionUnknown} + +func New(log logr.Logger) *Metrics { + var ( + certificateExpiryTimeSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "certificate_expiration_timestamp_seconds", + Help: "The date after which the certificate expires. Expressed as a Unix Epoch Time.", + }, + []string{"name", "namespace"}, + ) + + certificateReadyStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "certificate_ready_status", + Help: "The ready status of the certificate.", + }, + []string{"name", "namespace", "condition"}, + ) + + // acmeClientRequestCount is a Prometheus summary to collect the number of + // requests made to each endpoint with the ACME client. + acmeClientRequestCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "acme_client_request_count", + Help: "The number of requests made by the ACME client.", + Subsystem: "http", + }, + []string{"scheme", "host", "path", "method", "status"}, + ) + + // acmeClientRequestDurationSeconds is a Prometheus summary to collect request + // times for the ACME client. + acmeClientRequestDurationSeconds = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: namespace, + Name: "acme_client_request_duration_seconds", + Help: "The HTTP request latencies in seconds for the ACME client.", + Subsystem: "http", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, + []string{"scheme", "host", "path", "method", "status"}, + ) + + controllerSyncCallCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "controller_sync_call_count", + Help: "The number of sync() calls made by a controller.", + }, + []string{"controller"}, + ) + ) + router := mux.NewRouter() - // Create server and register prometheus metrics handler - s := &Metrics{ - ctx: ctx, - Server: http.Server{ + // Create server and register Prometheus metrics handler + m := &Metrics{ + log: log.WithName("metrics"), + registry: prometheus.NewRegistry(), + server: &http.Server{ Addr: prometheusMetricsServerAddress, ReadTimeout: prometheusMetricsServerReadTimeout, WriteTimeout: prometheusMetricsServerWriteTimeout, MaxHeaderBytes: prometheusMetricsServerMaxHeaderBytes, Handler: router, }, - activeCertificates: nil, - registry: prometheus.NewRegistry(), - CertificateExpiryTimeSeconds: CertificateExpiryTimeSeconds, - CertificateReadyStatus: CertificateReadyStatus, - ACMEClientRequestDurationSeconds: ACMEClientRequestDurationSeconds, - ACMEClientRequestCount: ACMEClientRequestCount, - ControllerSyncCallCount: ControllerSyncCallCount, + + registeredCertificates: make(map[string]struct{}), + + certificateExpiryTimeSeconds: certificateExpiryTimeSeconds, + certificateReadyStatus: certificateReadyStatus, + acmeClientRequestCount: acmeClientRequestCount, + acmeClientRequestDurationSeconds: acmeClientRequestDurationSeconds, + controllerSyncCallCount: controllerSyncCallCount, } - router.Handle("/metrics", promhttp.HandlerFor(s.registry, promhttp.HandlerOpts{})) + router.Handle("/metrics", promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})) - return s -} - -func (m *Metrics) waitShutdown(stopCh <-chan struct{}) { - log := logf.FromContext(m.ctx) - <-stopCh - log.Info("stopping Prometheus metrics server...") - - ctx, cancel := context.WithTimeout(context.Background(), prometheusMetricsServerShutdownTimeout) - defer cancel() - - if err := m.Shutdown(ctx); err != nil { - log.Error(err, "prometheus metrics server shutdown failed", err) - return - } - - log.Info("prometheus metrics server gracefully stopped") + return m } +// Start will register the Prometheu metrics, and start the Prometheus server func (m *Metrics) Start(stopCh <-chan struct{}) { - log := logf.FromContext(m.ctx) - - m.registry.MustRegister(m.CertificateExpiryTimeSeconds) - m.registry.MustRegister(m.CertificateReadyStatus) - m.registry.MustRegister(m.ACMEClientRequestDurationSeconds) - m.registry.MustRegister(m.ACMEClientRequestCount) - m.registry.MustRegister(m.ControllerSyncCallCount) + m.registry.MustRegister(m.certificateExpiryTimeSeconds) + m.registry.MustRegister(m.certificateReadyStatus) + m.registry.MustRegister(m.acmeClientRequestDurationSeconds) + m.registry.MustRegister(m.acmeClientRequestCount) + m.registry.MustRegister(m.controllerSyncCallCount) go func() { - log := log.WithValues("address", m.Addr) + log := m.log.WithValues("address", m.server.Addr) log.Info("listening for connections on") - if err := m.ListenAndServe(); err != nil { + if err := m.server.ListenAndServe(); err != nil { log.Error(err, "error running prometheus metrics server") return } log.Info("prometheus metrics server exited") - }() - // clean up metrics referring to deleted resources every minute - go wait.Until(func() { m.cleanUp() }, time.Minute, stopCh) - - m.waitShutdown(stopCh) + <-stopCh + m.shutdown() } -// UpdateCertificateExpiry updates the expiry time of a certificate -func (m *Metrics) UpdateCertificateExpiry(crt *v1alpha2.Certificate, secretLister corelisters.SecretLister) { - log := logf.FromContext(m.ctx) - log = logf.WithResource(log, crt) - log = logf.WithRelatedResourceName(log, crt.Spec.SecretName, crt.Namespace, "Secret") +// ObserveACMERequestDuration increases bucket counters for that ACME client duration. +func (m *Metrics) ObserveACMERequestDuration(duration time.Duration, labels ...string) { + m.acmeClientRequestDurationSeconds.WithLabelValues(labels...).Observe(duration.Seconds()) +} - log.V(logf.DebugLevel).Info("attempting to retrieve secret for certificate") - // grab existing certificate - cert, err := kube.SecretTLSCert(m.ctx, secretLister, crt.Namespace, crt.Spec.SecretName) +// IncrementACMERequestCount increases the acme client request counter. +func (m *Metrics) IncrementACMERequestCount(labels ...string) { + m.acmeClientRequestCount.WithLabelValues(labels...).Inc() +} + +// IncrementSyncCallCount will increase the sync counter for that controller. +func (m *Metrics) IncrementSyncCallCount(controllerName string) { + m.controllerSyncCallCount.WithLabelValues(controllerName).Inc() +} + +// UpdateCertificate will update that Certificate metric with expiry and Ready +// condition. +func (m *Metrics) UpdateCertificate(ctx context.Context, crt *cmapi.Certificate) { + key, err := cache.MetaNamespaceKeyFunc(crt) if err != nil { - if !apierrors.IsNotFound(err) && !errors.IsInvalidData(err) { - log.Error(err, "error reading secret for certificate") - } + log := logf.WithRelatedResource(m.log, crt) + log.Error(err, "failed to get key from certificate object") return } - updateX509Expiry(crt, cert) + m.updateCertificateStatus(key, crt) + m.updateCertificateExpiry(ctx, key, crt) } -func updateX509Expiry(crt *v1alpha2.Certificate, cert *x509.Certificate) { - expiryTime := cert.NotAfter - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt) - if err != nil { - return +// updateCertificateExpiry updates the expiry time of a certificate +func (m *Metrics) updateCertificateExpiry(ctx context.Context, key string, crt *cmapi.Certificate) { + expiryTime := 0.0 + + if crt.Status.NotAfter != nil { + expiryTime = float64(crt.Status.NotAfter.Unix()) } - registeredCertificates.mtx.Lock() - defer registeredCertificates.mtx.Unlock() - // set certificate expiry time - CertificateExpiryTimeSeconds.With(prometheus.Labels{ + m.certificateExpiryTimeSeconds.With(prometheus.Labels{ "name": crt.Name, - "namespace": crt.Namespace}).Set(float64(expiryTime.Unix())) - registerCertificateKey(key) + "namespace": crt.Namespace}).Set(expiryTime) + + m.mux.Lock() + m.registeredCertificates[key] = struct{}{} + m.mux.Unlock() } -func (m *Metrics) UpdateCertificateStatus(crt *v1alpha2.Certificate) { - log := logf.FromContext(m.ctx) - log = logf.WithResource(log, crt) +// updateCertificateStatus will update the metric for that Certificate +func (m *Metrics) updateCertificateStatus(key string, crt *cmapi.Certificate) { + defer func() { + m.mux.Lock() + m.registeredCertificates[key] = struct{}{} + m.mux.Unlock() + }() - log.V(logf.DebugLevel).Info("attempting to retrieve ready status for certificate") for _, c := range crt.Status.Conditions { - switch c.Type { - case v1alpha2.CertificateConditionReady: - updateCertificateReadyStatus(crt, c.Status) + if c.Type == cmapi.CertificateConditionReady { + m.updateCertificateReadyStatus(crt, c.Status) + return } } + + // If no status condition set yet, set to Unknown + m.updateCertificateReadyStatus(crt, cmmeta.ConditionUnknown) } -func updateCertificateReadyStatus(crt *v1alpha2.Certificate, current cmmeta.ConditionStatus) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt) - if err != nil { - return - } - - registeredCertificates.mtx.Lock() - defer registeredCertificates.mtx.Unlock() +func (m *Metrics) updateCertificateReadyStatus(crt *cmapi.Certificate, current cmmeta.ConditionStatus) { for _, condition := range readyConditionStatuses { value := 0.0 - if string(current) == condition { + + if current == condition { value = 1.0 } - CertificateReadyStatus.With(prometheus.Labels{ + + m.certificateReadyStatus.With(prometheus.Labels{ "name": crt.Name, "namespace": crt.Namespace, "condition": string(condition), }).Set(value) } - registerCertificateKey(key) } -// registerCertificateKey adds an entry in registeredCertificates to track -// which certificates have metrics stored in prometheus, allowing for easier -// clean-up. -// You MUST lock the mutex before calling this function, this ensures no other -// function is cleaning up while we are registering a certificate -func registerCertificateKey(key string) { - registeredCertificates.certificates[key] = struct{}{} -} +// RemoveCertificate will delete the Certificate metrics from continuing to be +// exposed. +func (m *Metrics) RemoveCertificate(key string) { + m.mux.Lock() + defer m.mux.Unlock() -func (m *Metrics) SetActiveCertificates(cl cmlisters.CertificateLister) { - m.activeCertificates = cl -} - -// cleanUp removes any metrics which reference resources which no longer exist -func (m *Metrics) cleanUp() { - log := logf.FromContext(m.ctx) - log.V(logf.DebugLevel).Info("attempting to clean up metrics for recently deleted certificates") - - if m.activeCertificates == nil { - log.V(logf.DebugLevel).Info("active certificates is still uninitialized") - return - } - - activeCrts, err := m.activeCertificates.List(labels.Everything()) - if err != nil { - log.Error(err, "error retrieving active certificates") - return - } - - cleanUpCertificates(activeCrts) -} - -// cleanUpCertificates removes metrics for recently deleted certificates -func cleanUpCertificates(activeCrts []*v1alpha2.Certificate) { - activeMap := make(map[string]struct{}, len(activeCrts)) - for _, crt := range activeCrts { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt) - if err != nil { - continue - } - - activeMap[key] = struct{}{} - } - - registeredCertificates.mtx.Lock() - defer registeredCertificates.mtx.Unlock() - var toCleanUp []string - for key := range registeredCertificates.certificates { - if _, found := activeMap[key]; !found { - toCleanUp = append(toCleanUp, key) - } - } - - for _, key := range toCleanUp { - cleanUpCertificateByKey(key) - } -} - -// metricCleanUpCertificate creates a clean up function which deletes the entry -// (if any) for a certificate in the given metric -func metricCleanUpCertificate(c cleanableMetric) func(string, string) { - return func(name, namespace string) { - c.DeleteLabelValues(name, namespace) - } -} - -// metricCleanUpCertificateWith creates a clean up function which deletes the -// entries (if any) for a certificate in the given metric, iterating over the -// additional labels. -// This is used if the metric keys on data in addition to the name and -// namespace. -func metricCleanUpCertificateWith(c cleanableMetric, additionalLabels []string) func(string, string) { - return func(name, namespace string) { - for _, label := range additionalLabels { - c.DeleteLabelValues(name, namespace, label) - } - } -} - -// cleanUpCertificateByKey removes metrics which refer to a certificate, -// given the key of the certificate. -func cleanUpCertificateByKey(key string) { namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { + m.log.Error(err, "failed to get namespace and name from key") return } - // apply all the clean up functions - for _, f := range cleanUpFunctions { - f(name, namespace) + // If the certificate is not registered, exit early + if _, ok := m.registeredCertificates[key]; !ok { + return } - delete(registeredCertificates.certificates, key) + m.certificateExpiryTimeSeconds.DeleteLabelValues(name, namespace) + for _, condition := range readyConditionStatuses { + m.certificateReadyStatus.DeleteLabelValues(name, namespace, string(condition)) + } + + delete(m.registeredCertificates, key) } -func (m *Metrics) IncrementSyncCallCount(controllerName string) { - log := logf.FromContext(m.ctx) - log.V(logf.DebugLevel).Info("incrementing controller sync call count", "controllerName", controllerName) - ControllerSyncCallCount.WithLabelValues(controllerName).Inc() +func (m *Metrics) shutdown() { + m.log.Info("stopping Prometheus metrics server...") + + ctx, cancel := context.WithTimeout(context.Background(), prometheusMetricsServerShutdownTimeout) + defer cancel() + + if err := m.server.Shutdown(ctx); err != nil { + m.log.Error(err, "prometheus metrics server shutdown failed", err) + return + } + + m.log.Info("prometheus metrics server gracefully stopped") } diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 5dea94350..42964b847 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Jetstack cert-manager contributors. +Copyright 2020 The Jetstack cert-manager contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ limitations under the License. package metrics import ( - "crypto/x509" + "context" "strings" "testing" "time" @@ -25,124 +25,118 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2" + cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2" cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1" + logtesting "github.com/jetstack/cert-manager/pkg/logs/testing" + "github.com/jetstack/cert-manager/test/unit/gen" ) -func getReadyConditionStatus(crt *v1alpha2.Certificate) cmmeta.ConditionStatus { - for _, c := range crt.Status.Conditions { - switch c.Type { - case v1alpha2.CertificateConditionReady: - return c.Status - } - } - return cmmeta.ConditionUnknown -} - -func buildCertificate(name, namespace string, condition cmmeta.ConditionStatus) *v1alpha2.Certificate { - return &v1alpha2.Certificate{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Status: v1alpha2.CertificateStatus{ - Conditions: []v1alpha2.CertificateCondition{ - { - Type: v1alpha2.CertificateConditionReady, - Status: condition, - }, - }, - }, - } -} - -func TestUpdateCertificateExpiry(t *testing.T) { - const metadata = ` +const expiryMetadata = ` # HELP certmanager_certificate_expiration_timestamp_seconds The date after which the certificate expires. Expressed as a Unix Epoch Time. # TYPE certmanager_certificate_expiration_timestamp_seconds gauge ` + +const readyMetadata = ` + # HELP certmanager_certificate_ready_status The ready status of the certificate. + # TYPE certmanager_certificate_ready_status gauge +` + +func TestCertificateMetrics(t *testing.T) { type testT struct { - crt *v1alpha2.Certificate - cert *x509.Certificate - expected string + crt *cmapi.Certificate + expectedExpiry, expectedReady string } tests := map[string]testT{ - "first": { - crt: &v1alpha2.Certificate{ - ObjectMeta: metav1.ObjectMeta{ - Name: "something", - Namespace: "default", - }, - }, - cert: &x509.Certificate{ - // fixed expiry time for testing - NotAfter: time.Unix(2208988804, 0), - }, - expected: ` - certmanager_certificate_expiration_timestamp_seconds{name="something",namespace="default"} 2.208988804e+09 + "certificate with expiry and ready status": { + crt: gen.Certificate("test-certificate", + gen.SetCertificateNamespace("test-ns"), + gen.SetCertificateNotAfter(metav1.Time{ + Time: time.Unix(2208988804, 0), + }), + gen.SetCertificateStatusCondition(cmapi.CertificateCondition{ + Type: cmapi.CertificateConditionReady, + Status: cmmeta.ConditionTrue, + }), + ), + expectedExpiry: ` + certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 2.208988804e+09 +`, + expectedReady: ` + certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 0 + certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 1 + certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 0 +`, + }, + + "certificate with no expiry and no status should give an expiry of 0 and Unknown status": { + crt: gen.Certificate("test-certificate", + gen.SetCertificateNamespace("test-ns"), + ), + expectedExpiry: ` + certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 0 +`, + expectedReady: ` + certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 0 + certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 0 + certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 1 +`, + }, + + "certificate with expiry and status False should give an expiry and False status": { + crt: gen.Certificate("test-certificate", + gen.SetCertificateNamespace("test-ns"), + gen.SetCertificateNotAfter(metav1.Time{ + Time: time.Unix(100, 0), + }), + gen.SetCertificateStatusCondition(cmapi.CertificateCondition{ + Type: cmapi.CertificateConditionReady, + Status: cmmeta.ConditionFalse, + }), + ), + expectedExpiry: ` + certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 100 +`, + expectedReady: ` + certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 1 + certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 0 + certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 0 +`, + }, + "certificate with expiry and status Unknown should give an expiry and Unknown status": { + crt: gen.Certificate("test-certificate", + gen.SetCertificateNamespace("test-ns"), + gen.SetCertificateNotAfter(metav1.Time{ + Time: time.Unix(99999, 0), + }), + gen.SetCertificateStatusCondition(cmapi.CertificateCondition{ + Type: cmapi.CertificateConditionReady, + Status: cmmeta.ConditionUnknown, + }), + ), + expectedExpiry: ` + certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 99999 +`, + expectedReady: ` + certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 0 + certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 0 + certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 1 `, }, } for n, test := range tests { t.Run(n, func(t *testing.T) { - defer cleanUpCertificates(nil) + m := New(logtesting.TestLogger{T: t}) + m.UpdateCertificate(context.TODO(), test.crt) - updateX509Expiry(test.crt, test.cert) - - if err := testutil.CollectAndCompare( - CertificateExpiryTimeSeconds, - strings.NewReader(metadata+test.expected), + if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds, + strings.NewReader(expiryMetadata+test.expectedExpiry), "certmanager_certificate_expiration_timestamp_seconds", ); err != nil { t.Errorf("unexpected collecting result:\n%s", err) } - }) - } -} -func TestUpdateCertificateReadyStatus(t *testing.T) { - const metadata = ` - # HELP certmanager_certificate_ready_status The ready status of the certificate. - # TYPE certmanager_certificate_ready_status gauge -` - - type testT struct { - crt *v1alpha2.Certificate - expected string - } - tests := map[string]testT{ - "ready status true is updated correctly": { - crt: buildCertificate("something", "default", cmmeta.ConditionTrue), - expected: ` - certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 0 - certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 1 - certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 0 -`, - }, - "ready status false is updated correctly": { - crt: buildCertificate("something", "default", cmmeta.ConditionFalse), - expected: ` - certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 1 - certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 0 - certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 0 -`, - }, - "ready status unknown is updated correctly": { - crt: buildCertificate("something", "default", cmmeta.ConditionUnknown), - expected: ` - certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 0 - certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 0 - certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 1 -`, - }, - } - for n, test := range tests { - t.Run(n, func(t *testing.T) { - updateCertificateReadyStatus(test.crt, getReadyConditionStatus(test.crt)) - - if err := testutil.CollectAndCompare( - CertificateReadyStatus, - strings.NewReader(metadata+test.expected), + if err := testutil.CollectAndCompare(m.certificateReadyStatus, + strings.NewReader(readyMetadata+test.expectedReady), "certmanager_certificate_ready_status", ); err != nil { t.Errorf("unexpected collecting result:\n%s", err) @@ -151,118 +145,112 @@ func TestUpdateCertificateReadyStatus(t *testing.T) { } } -func TestCleanUp(t *testing.T) { - const metadataExpiry = ` - # HELP certmanager_certificate_expiration_timestamp_seconds The date after which the certificate expires. Expressed as a Unix Epoch Time. - # TYPE certmanager_certificate_expiration_timestamp_seconds gauge -` +func TestCertificateCache(t *testing.T) { + m := New(logtesting.TestLogger{T: t}) - const metadataReady = ` - # HELP certmanager_certificate_ready_status The ready status of the certificate. - # TYPE certmanager_certificate_ready_status gauge -` - type testT struct { - active map[*v1alpha2.Certificate]*x509.Certificate - inactive map[*v1alpha2.Certificate]*x509.Certificate - expectedExpiry string - expectedReady string + crt1 := gen.Certificate("crt1", + gen.SetCertificateUID("uid-1"), + gen.SetCertificateNotAfter(metav1.Time{ + Time: time.Unix(100, 0), + }), + gen.SetCertificateStatusCondition(cmapi.CertificateCondition{ + Type: cmapi.CertificateConditionReady, + Status: cmmeta.ConditionUnknown, + }), + ) + crt2 := gen.Certificate("crt2", + gen.SetCertificateUID("uid-2"), + gen.SetCertificateNotAfter(metav1.Time{ + Time: time.Unix(200, 0), + }), + gen.SetCertificateStatusCondition(cmapi.CertificateCondition{ + Type: cmapi.CertificateConditionReady, + Status: cmmeta.ConditionTrue, + }), + ) + crt3 := gen.Certificate("crt3", + gen.SetCertificateUID("uid-3"), + gen.SetCertificateNotAfter(metav1.Time{ + Time: time.Unix(300, 0), + }), + gen.SetCertificateStatusCondition(cmapi.CertificateCondition{ + Type: cmapi.CertificateConditionReady, + Status: cmmeta.ConditionFalse, + }), + ) + + // Observe all three Certificate metrics + m.UpdateCertificate(context.TODO(), crt1) + m.UpdateCertificate(context.TODO(), crt2) + m.UpdateCertificate(context.TODO(), crt3) + + // Check all three metrics exist + if err := testutil.CollectAndCompare(m.certificateReadyStatus, + strings.NewReader(readyMetadata+` + certmanager_certificate_ready_status{condition="False",name="crt1",namespace="default-unit-test-ns"} 0 + certmanager_certificate_ready_status{condition="False",name="crt2",namespace="default-unit-test-ns"} 0 + certmanager_certificate_ready_status{condition="False",name="crt3",namespace="default-unit-test-ns"} 1 + certmanager_certificate_ready_status{condition="True",name="crt1",namespace="default-unit-test-ns"} 0 + certmanager_certificate_ready_status{condition="True",name="crt2",namespace="default-unit-test-ns"} 1 + certmanager_certificate_ready_status{condition="True",name="crt3",namespace="default-unit-test-ns"} 0 + certmanager_certificate_ready_status{condition="Unknown",name="crt1",namespace="default-unit-test-ns"} 1 + certmanager_certificate_ready_status{condition="Unknown",name="crt2",namespace="default-unit-test-ns"} 0 + certmanager_certificate_ready_status{condition="Unknown",name="crt3",namespace="default-unit-test-ns"} 0 +`), + "certmanager_certificate_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) } - tests := map[string]testT{ - "inactive certificate metrics cleaned up while active certificate metrics kept": { - active: map[*v1alpha2.Certificate]*x509.Certificate{ - buildCertificate("active", "default", cmmeta.ConditionTrue): { - // fixed expiry time for testing - NotAfter: time.Unix(2208988804, 0), - }, - }, - inactive: map[*v1alpha2.Certificate]*x509.Certificate{ - buildCertificate("inactive", "default", cmmeta.ConditionTrue): { - // fixed expiry time for testing - NotAfter: time.Unix(2208988804, 0), - }, - }, - expectedExpiry: ` - certmanager_certificate_expiration_timestamp_seconds{name="active",namespace="default"} 2.208988804e+09 -`, - expectedReady: ` - certmanager_certificate_ready_status{condition="False",name="active",namespace="default"} 0 - certmanager_certificate_ready_status{condition="True",name="active",namespace="default"} 1 - certmanager_certificate_ready_status{condition="Unknown",name="active",namespace="default"} 0 -`, - }, - "no metrics cleaned up when only active certificate metrics": { - active: map[*v1alpha2.Certificate]*x509.Certificate{ - buildCertificate("active", "default", cmmeta.ConditionTrue): { - // fixed expiry time for testing - NotAfter: time.Unix(2208988804, 0), - }, - buildCertificate("also-active", "default", cmmeta.ConditionTrue): { - // fixed expiry time for testing - NotAfter: time.Unix(2208988804, 0), - }, - }, - inactive: map[*v1alpha2.Certificate]*x509.Certificate{}, - expectedExpiry: ` - certmanager_certificate_expiration_timestamp_seconds{name="active",namespace="default"} 2.208988804e+09 - certmanager_certificate_expiration_timestamp_seconds{name="also-active",namespace="default"} 2.208988804e+09 -`, - expectedReady: ` - certmanager_certificate_ready_status{condition="False",name="active",namespace="default"} 0 - certmanager_certificate_ready_status{condition="False",name="also-active",namespace="default"} 0 - certmanager_certificate_ready_status{condition="True",name="active",namespace="default"} 1 - certmanager_certificate_ready_status{condition="True",name="also-active",namespace="default"} 1 - certmanager_certificate_ready_status{condition="Unknown",name="active",namespace="default"} 0 - certmanager_certificate_ready_status{condition="Unknown",name="also-active",namespace="default"} 0 -`, - }, - "all metrics cleaned up when only inactive certificate metrics": { - active: map[*v1alpha2.Certificate]*x509.Certificate{}, - inactive: map[*v1alpha2.Certificate]*x509.Certificate{ - buildCertificate("inactive", "default", cmmeta.ConditionTrue): { - // fixed expiry time for testing - NotAfter: time.Unix(2208988804, 0), - }, - buildCertificate("also-inactive", "default", cmmeta.ConditionTrue): { - // fixed expiry time for testing - NotAfter: time.Unix(2208988804, 0), - }, - }, - expectedExpiry: "", - expectedReady: "", - }, + if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds, + strings.NewReader(expiryMetadata+` + certmanager_certificate_expiration_timestamp_seconds{name="crt1",namespace="default-unit-test-ns"} 100 + certmanager_certificate_expiration_timestamp_seconds{name="crt2",namespace="default-unit-test-ns"} 200 + certmanager_certificate_expiration_timestamp_seconds{name="crt3",namespace="default-unit-test-ns"} 300 +`), + "certmanager_certificate_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) } - for n, test := range tests { - t.Run(n, func(t *testing.T) { - defer cleanUpCertificates(nil) - var activeCrts []*v1alpha2.Certificate - for crt, cert := range test.active { - updateX509Expiry(crt, cert) - updateCertificateReadyStatus(crt, getReadyConditionStatus(crt)) - activeCrts = append(activeCrts, crt) - } - for crt, cert := range test.inactive { - updateCertificateReadyStatus(crt, getReadyConditionStatus(crt)) - updateX509Expiry(crt, cert) - } + // Remove second certificate and check not exists + m.RemoveCertificate("default-unit-test-ns/crt2") + if err := testutil.CollectAndCompare(m.certificateReadyStatus, + strings.NewReader(readyMetadata+` + certmanager_certificate_ready_status{condition="False",name="crt1",namespace="default-unit-test-ns"} 0 + certmanager_certificate_ready_status{condition="False",name="crt3",namespace="default-unit-test-ns"} 1 + certmanager_certificate_ready_status{condition="True",name="crt1",namespace="default-unit-test-ns"} 0 + certmanager_certificate_ready_status{condition="True",name="crt3",namespace="default-unit-test-ns"} 0 + certmanager_certificate_ready_status{condition="Unknown",name="crt1",namespace="default-unit-test-ns"} 1 + certmanager_certificate_ready_status{condition="Unknown",name="crt3",namespace="default-unit-test-ns"} 0 +`), + "certmanager_certificate_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds, + strings.NewReader(expiryMetadata+` + certmanager_certificate_expiration_timestamp_seconds{name="crt1",namespace="default-unit-test-ns"} 100 + certmanager_certificate_expiration_timestamp_seconds{name="crt3",namespace="default-unit-test-ns"} 300 +`), + "certmanager_certificate_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } - cleanUpCertificates(activeCrts) - - if err := testutil.CollectAndCompare( - CertificateExpiryTimeSeconds, - strings.NewReader(metadataExpiry+test.expectedExpiry), - "certmanager_certificate_expiration_timestamp_seconds", - ); err != nil { - t.Errorf("unexpected collecting result:\n%s", err) - } - - if err := testutil.CollectAndCompare( - CertificateReadyStatus, - strings.NewReader(metadataReady+test.expectedReady), - "certmanager_certificate_ready_status", - ); err != nil { - t.Errorf("unexpected collecting result:\n%s", err) - } - }) + // Remove all Certificates (even is already removed) and observe no Certificates + m.RemoveCertificate("default-unit-test-ns/crt1") + m.RemoveCertificate("default-unit-test-ns/crt2") + m.RemoveCertificate("default-unit-test-ns/crt3") + if err := testutil.CollectAndCompare(m.certificateReadyStatus, + strings.NewReader(readyMetadata), + "certmanager_certificate_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds, + strings.NewReader(expiryMetadata), + "certmanager_certificate_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) } }