weiguoqiang/cert-manager
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Updates metrics package to be better consumable

Browse Source 
Signed-off-by: JoshVanL <vleeuwenjoshua@gmail.com>
This commit is contained in:
JoshVanL 2020-05-18 17:41:14 +01:00
parent 6a6f3dbf7a
commit 9e98d7b948
No known key found for this signature in database
GPG Key ID: E7A7196576A219DA
3 changed files with 381 additions and 487 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/metrics/BUILD.bazel
View File

@ -8,17 +8,11 @@ go_library(
deps = [ deps = [
"//pkg/apis/certmanager/v1alpha2:go_default_library", "//pkg/apis/certmanager/v1alpha2:go_default_library",
"//pkg/apis/meta/v1:go_default_library", "//pkg/apis/meta/v1:go_default_library",
"//pkg/client/listers/certmanager/v1alpha2:go_default_library",
"//pkg/logs:go_default_library", "//pkg/logs:go_default_library",
"//pkg/util/errors:go_default_library", "@com_github_go_logr_logr//:go_default_library",
"//pkg/util/kube:go_default_library",
"@com_github_gorilla_mux//:go_default_library", "@com_github_gorilla_mux//:go_default_library",
"@com_github_prometheus_client_golang//prometheus:go_default_library", "@com_github_prometheus_client_golang//prometheus:go_default_library",
"@com_github_prometheus_client_golang//prometheus/promhttp:go_default_library", "@com_github_prometheus_client_golang//prometheus/promhttp:go_default_library",
"@io_k8s_apimachinery//pkg/api/errors:go_default_library",
"@io_k8s_apimachinery//pkg/labels:go_default_library",
"@io_k8s_apimachinery//pkg/util/wait:go_default_library",
"@io_k8s_client_go//listers/core/v1:go_default_library",
"@io_k8s_client_go//tools/cache:go_default_library", "@io_k8s_client_go//tools/cache:go_default_library",
], ],
) )
@ -44,6 +38,8 @@ go_test(
deps = [ deps = [
"//pkg/apis/certmanager/v1alpha2:go_default_library", "//pkg/apis/certmanager/v1alpha2:go_default_library",
"//pkg/apis/meta/v1:go_default_library", "//pkg/apis/meta/v1:go_default_library",
"//pkg/logs/testing:go_default_library",
"//test/unit/gen:go_default_library",
"@com_github_prometheus_client_golang//prometheus/testutil:go_default_library", "@com_github_prometheus_client_golang//prometheus/testutil:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
], ],

454
pkg/metrics/metrics.go
View File

@ -1,5 +1,5 @@
/* /*
Copyright 2019 The Jetstack cert-manager contributors. Copyright 2020 The Jetstack cert-manager contributors.
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
@ -18,370 +18,280 @@ limitations under the License.
// cert-manager exposes the following metrics: // cert-manager exposes the following metrics:
// certificate_expiration_timestamp_seconds{name, namespace} // certificate_expiration_timestamp_seconds{name, namespace}
// certificate_ready_status{name, namespace, condition} // certificate_ready_status{name, namespace, condition}
// acme_client_request_count{"scheme", "host", "path", "method", "status"}
// acme_client_request_duration_seconds{"scheme", "host", "path", "method", "status"}
// controller_sync_call_count{"controller"}
package metrics package metrics
import ( import (
"context" "context"
"crypto/x509"
"net/http" "net/http"
"sync" "sync"
"time" "time"
"github.com/go-logr/logr"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/wait"
corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/cache"
"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2" cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1" cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
cmlisters "github.com/jetstack/cert-manager/pkg/client/listers/certmanager/v1alpha2"
logf "github.com/jetstack/cert-manager/pkg/logs" logf "github.com/jetstack/cert-manager/pkg/logs"
"github.com/jetstack/cert-manager/pkg/util/errors"
"github.com/jetstack/cert-manager/pkg/util/kube"
) )
const ( const (
// Namespace is the namespace for cert-manager metric names // Namespace is the namespace for cert-manager metric names
namespace = "certmanager" namespace = "certmanager"
prometheusMetricsServerAddress = "0.0.0.0:9402" prometheusMetricsServerAddress = "127.0.0.1:9402"
prometheusMetricsServerShutdownTimeout = 5 * time.Second prometheusMetricsServerShutdownTimeout = 5 * time.Second
prometheusMetricsServerReadTimeout = 8 * time.Second prometheusMetricsServerReadTimeout = 8 * time.Second
prometheusMetricsServerWriteTimeout = 8 * time.Second prometheusMetricsServerWriteTimeout = 8 * time.Second
prometheusMetricsServerMaxHeaderBytes = 1 << 20 // 1 MiB prometheusMetricsServerMaxHeaderBytes = 1 << 20 // 1 MiB
) )
var readyConditionStatuses = [...]string{string(cmmeta.ConditionTrue), string(cmmeta.ConditionFalse), string(cmmeta.ConditionUnknown)} // Metrics is designed to be a shared object for updating the metrics exposed
// by cert-manager
// Default set of metrics
var Default = New(logf.NewContext(context.Background(), logf.Log.WithName("metrics")))
var CertificateExpiryTimeSeconds = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: namespace,
Name: "certificate_expiration_timestamp_seconds",
Help: "The date after which the certificate expires. Expressed as a Unix Epoch Time.",
},
[]string{"name", "namespace"},
)
var CertificateReadyStatus = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: namespace,
Name: "certificate_ready_status",
Help: "The ready status of the certificate.",
},
[]string{"name", "namespace", "condition"},
)
// ACMEClientRequestCount is a Prometheus summary to collect the number of
// requests made to each endpoint with the ACME client.
var ACMEClientRequestCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Name: "acme_client_request_count",
Help: "The number of requests made by the ACME client.",
Subsystem: "http",
},
[]string{"scheme", "host", "path", "method", "status"},
)
// ACMEClientRequestDurationSeconds is a Prometheus summary to collect request
// times for the ACME client.
var ACMEClientRequestDurationSeconds = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: namespace,
Name: "acme_client_request_duration_seconds",
Help: "The HTTP request latencies in seconds for the ACME client.",
Subsystem: "http",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
[]string{"scheme", "host", "path", "method", "status"},
)
var ControllerSyncCallCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Name: "controller_sync_call_count",
Help: "The number of sync() calls made by a controller.",
},
[]string{"controller"},
)
// registeredCertificates holds the set of all certificates which are currently
// registered by Prometheus
var registeredCertificates = &struct {
certificates map[string]struct{}
mtx sync.Mutex
}{
certificates: make(map[string]struct{}),
}
// cleanUpFunctions are functions called to clean up metrics which refer to
// deleted certificates, inputs are name and namespace of the certificate
var cleanUpFunctions = []func(string, string){
metricCleanUpCertificate(CertificateExpiryTimeSeconds),
metricCleanUpCertificateWith(CertificateReadyStatus, readyConditionStatuses[:]),
}
type cleanableMetric interface {
DeleteLabelValues(...string) bool
}
type Metrics struct { type Metrics struct {
ctx context.Context log logr.Logger
http.Server registry *prometheus.Registry
activeCertificates cmlisters.CertificateLister server *http.Server
// TODO (@dippynark): switch this to use an interface to make it testable mux sync.Mutex
registry *prometheus.Registry
CertificateExpiryTimeSeconds *prometheus.GaugeVec certificateExpiryTimeSeconds *prometheus.GaugeVec
CertificateReadyStatus *prometheus.GaugeVec certificateReadyStatus *prometheus.GaugeVec
ACMEClientRequestDurationSeconds *prometheus.SummaryVec acmeClientRequestDurationSeconds *prometheus.SummaryVec
ACMEClientRequestCount *prometheus.CounterVec acmeClientRequestCount *prometheus.CounterVec
ControllerSyncCallCount *prometheus.CounterVec controllerSyncCallCount *prometheus.CounterVec
registeredCertificates map[string]struct{}
} }
func New(ctx context.Context) *Metrics { var readyConditionStatuses = [...]cmmeta.ConditionStatus{cmmeta.ConditionTrue, cmmeta.ConditionFalse, cmmeta.ConditionUnknown}
func New(log logr.Logger) *Metrics {
var (
certificateExpiryTimeSeconds = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: namespace,
Name: "certificate_expiration_timestamp_seconds",
Help: "The date after which the certificate expires. Expressed as a Unix Epoch Time.",
},
[]string{"name", "namespace"},
)
certificateReadyStatus = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: namespace,
Name: "certificate_ready_status",
Help: "The ready status of the certificate.",
},
[]string{"name", "namespace", "condition"},
)
// acmeClientRequestCount is a Prometheus summary to collect the number of
// requests made to each endpoint with the ACME client.
acmeClientRequestCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Name: "acme_client_request_count",
Help: "The number of requests made by the ACME client.",
Subsystem: "http",
},
[]string{"scheme", "host", "path", "method", "status"},
)
// acmeClientRequestDurationSeconds is a Prometheus summary to collect request
// times for the ACME client.
acmeClientRequestDurationSeconds = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: namespace,
Name: "acme_client_request_duration_seconds",
Help: "The HTTP request latencies in seconds for the ACME client.",
Subsystem: "http",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
[]string{"scheme", "host", "path", "method", "status"},
)
controllerSyncCallCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Name: "controller_sync_call_count",
Help: "The number of sync() calls made by a controller.",
},
[]string{"controller"},
)
)
router := mux.NewRouter() router := mux.NewRouter()
// Create server and register prometheus metrics handler // Create server and register Prometheus metrics handler
s := &Metrics{ m := &Metrics{
ctx: ctx, log: log.WithName("metrics"),
Server: http.Server{ registry: prometheus.NewRegistry(),
server: &http.Server{
Addr: prometheusMetricsServerAddress, Addr: prometheusMetricsServerAddress,
ReadTimeout: prometheusMetricsServerReadTimeout, ReadTimeout: prometheusMetricsServerReadTimeout,
WriteTimeout: prometheusMetricsServerWriteTimeout, WriteTimeout: prometheusMetricsServerWriteTimeout,
MaxHeaderBytes: prometheusMetricsServerMaxHeaderBytes, MaxHeaderBytes: prometheusMetricsServerMaxHeaderBytes,
Handler: router, Handler: router,
}, },
activeCertificates: nil,
registry: prometheus.NewRegistry(), registeredCertificates: make(map[string]struct{}),
CertificateExpiryTimeSeconds: CertificateExpiryTimeSeconds,
CertificateReadyStatus: CertificateReadyStatus, certificateExpiryTimeSeconds: certificateExpiryTimeSeconds,
ACMEClientRequestDurationSeconds: ACMEClientRequestDurationSeconds, certificateReadyStatus: certificateReadyStatus,
ACMEClientRequestCount: ACMEClientRequestCount, acmeClientRequestCount: acmeClientRequestCount,
ControllerSyncCallCount: ControllerSyncCallCount, acmeClientRequestDurationSeconds: acmeClientRequestDurationSeconds,
controllerSyncCallCount: controllerSyncCallCount,
} }
router.Handle("/metrics", promhttp.HandlerFor(s.registry, promhttp.HandlerOpts{})) router.Handle("/metrics", promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{}))
return s return m
}
func (m *Metrics) waitShutdown(stopCh <-chan struct{}) {
log := logf.FromContext(m.ctx)
<-stopCh
log.Info("stopping Prometheus metrics server...")
ctx, cancel := context.WithTimeout(context.Background(), prometheusMetricsServerShutdownTimeout)
defer cancel()
if err := m.Shutdown(ctx); err != nil {
log.Error(err, "prometheus metrics server shutdown failed", err)
return
}
log.Info("prometheus metrics server gracefully stopped")
} }
// Start will register the Prometheu metrics, and start the Prometheus server
func (m *Metrics) Start(stopCh <-chan struct{}) { func (m *Metrics) Start(stopCh <-chan struct{}) {
log := logf.FromContext(m.ctx) m.registry.MustRegister(m.certificateExpiryTimeSeconds)
m.registry.MustRegister(m.certificateReadyStatus)
m.registry.MustRegister(m.CertificateExpiryTimeSeconds) m.registry.MustRegister(m.acmeClientRequestDurationSeconds)
m.registry.MustRegister(m.CertificateReadyStatus) m.registry.MustRegister(m.acmeClientRequestCount)
m.registry.MustRegister(m.ACMEClientRequestDurationSeconds) m.registry.MustRegister(m.controllerSyncCallCount)
m.registry.MustRegister(m.ACMEClientRequestCount)
m.registry.MustRegister(m.ControllerSyncCallCount)
go func() { go func() {
log := log.WithValues("address", m.Addr) log := m.log.WithValues("address", m.server.Addr)
log.Info("listening for connections on") log.Info("listening for connections on")
if err := m.ListenAndServe(); err != nil { if err := m.server.ListenAndServe(); err != nil {
log.Error(err, "error running prometheus metrics server") log.Error(err, "error running prometheus metrics server")
return return
} }
log.Info("prometheus metrics server exited") log.Info("prometheus metrics server exited")
}() }()
// clean up metrics referring to deleted resources every minute <-stopCh
go wait.Until(func() { m.cleanUp() }, time.Minute, stopCh) m.shutdown()
m.waitShutdown(stopCh)
} }
// UpdateCertificateExpiry updates the expiry time of a certificate // ObserveACMERequestDuration increases bucket counters for that ACME client duration.
func (m *Metrics) UpdateCertificateExpiry(crt *v1alpha2.Certificate, secretLister corelisters.SecretLister) { func (m *Metrics) ObserveACMERequestDuration(duration time.Duration, labels ...string) {
log := logf.FromContext(m.ctx) m.acmeClientRequestDurationSeconds.WithLabelValues(labels...).Observe(duration.Seconds())
log = logf.WithResource(log, crt) }
log = logf.WithRelatedResourceName(log, crt.Spec.SecretName, crt.Namespace, "Secret")
log.V(logf.DebugLevel).Info("attempting to retrieve secret for certificate") // IncrementACMERequestCount increases the acme client request counter.
// grab existing certificate func (m *Metrics) IncrementACMERequestCount(labels ...string) {
cert, err := kube.SecretTLSCert(m.ctx, secretLister, crt.Namespace, crt.Spec.SecretName) m.acmeClientRequestCount.WithLabelValues(labels...).Inc()
}
// IncrementSyncCallCount will increase the sync counter for that controller.
func (m *Metrics) IncrementSyncCallCount(controllerName string) {
m.controllerSyncCallCount.WithLabelValues(controllerName).Inc()
}
// UpdateCertificate will update that Certificate metric with expiry and Ready
// condition.
func (m *Metrics) UpdateCertificate(ctx context.Context, crt *cmapi.Certificate) {
key, err := cache.MetaNamespaceKeyFunc(crt)
if err != nil { if err != nil {
if !apierrors.IsNotFound(err) && !errors.IsInvalidData(err) { log := logf.WithRelatedResource(m.log, crt)
log.Error(err, "error reading secret for certificate") log.Error(err, "failed to get key from certificate object")
}
return return
} }
updateX509Expiry(crt, cert) m.updateCertificateStatus(key, crt)
m.updateCertificateExpiry(ctx, key, crt)
} }
func updateX509Expiry(crt *v1alpha2.Certificate, cert *x509.Certificate) { // updateCertificateExpiry updates the expiry time of a certificate
expiryTime := cert.NotAfter func (m *Metrics) updateCertificateExpiry(ctx context.Context, key string, crt *cmapi.Certificate) {
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt) expiryTime := 0.0
if err != nil {
return if crt.Status.NotAfter != nil {
expiryTime = float64(crt.Status.NotAfter.Unix())
} }
registeredCertificates.mtx.Lock() m.certificateExpiryTimeSeconds.With(prometheus.Labels{
defer registeredCertificates.mtx.Unlock()
// set certificate expiry time
CertificateExpiryTimeSeconds.With(prometheus.Labels{
"name": crt.Name, "name": crt.Name,
"namespace": crt.Namespace}).Set(float64(expiryTime.Unix())) "namespace": crt.Namespace}).Set(expiryTime)
registerCertificateKey(key)
m.mux.Lock()
m.registeredCertificates[key] = struct{}{}
m.mux.Unlock()
} }
func (m *Metrics) UpdateCertificateStatus(crt *v1alpha2.Certificate) { // updateCertificateStatus will update the metric for that Certificate
log := logf.FromContext(m.ctx) func (m *Metrics) updateCertificateStatus(key string, crt *cmapi.Certificate) {
log = logf.WithResource(log, crt) defer func() {
m.mux.Lock()
m.registeredCertificates[key] = struct{}{}
m.mux.Unlock()
}()
log.V(logf.DebugLevel).Info("attempting to retrieve ready status for certificate")
for _, c := range crt.Status.Conditions { for _, c := range crt.Status.Conditions {
switch c.Type { if c.Type == cmapi.CertificateConditionReady {
case v1alpha2.CertificateConditionReady: m.updateCertificateReadyStatus(crt, c.Status)
updateCertificateReadyStatus(crt, c.Status) return
} }
} }
// If no status condition set yet, set to Unknown
m.updateCertificateReadyStatus(crt, cmmeta.ConditionUnknown)
} }
func updateCertificateReadyStatus(crt *v1alpha2.Certificate, current cmmeta.ConditionStatus) { func (m *Metrics) updateCertificateReadyStatus(crt *cmapi.Certificate, current cmmeta.ConditionStatus) {
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt)
if err != nil {
return
}
registeredCertificates.mtx.Lock()
defer registeredCertificates.mtx.Unlock()
for _, condition := range readyConditionStatuses { for _, condition := range readyConditionStatuses {
value := 0.0 value := 0.0
if string(current) == condition {
if current == condition {
value = 1.0 value = 1.0
} }
CertificateReadyStatus.With(prometheus.Labels{
m.certificateReadyStatus.With(prometheus.Labels{
"name": crt.Name, "name": crt.Name,
"namespace": crt.Namespace, "namespace": crt.Namespace,
"condition": string(condition), "condition": string(condition),
}).Set(value) }).Set(value)
} }
registerCertificateKey(key)
} }
// registerCertificateKey adds an entry in registeredCertificates to track // RemoveCertificate will delete the Certificate metrics from continuing to be
// which certificates have metrics stored in prometheus, allowing for easier // exposed.
// clean-up. func (m *Metrics) RemoveCertificate(key string) {
// You MUST lock the mutex before calling this function, this ensures no other m.mux.Lock()
// function is cleaning up while we are registering a certificate defer m.mux.Unlock()
func registerCertificateKey(key string) {
registeredCertificates.certificates[key] = struct{}{}
}
func (m *Metrics) SetActiveCertificates(cl cmlisters.CertificateLister) {
m.activeCertificates = cl
}
// cleanUp removes any metrics which reference resources which no longer exist
func (m *Metrics) cleanUp() {
log := logf.FromContext(m.ctx)
log.V(logf.DebugLevel).Info("attempting to clean up metrics for recently deleted certificates")
if m.activeCertificates == nil {
log.V(logf.DebugLevel).Info("active certificates is still uninitialized")
return
}
activeCrts, err := m.activeCertificates.List(labels.Everything())
if err != nil {
log.Error(err, "error retrieving active certificates")
return
}
cleanUpCertificates(activeCrts)
}
// cleanUpCertificates removes metrics for recently deleted certificates
func cleanUpCertificates(activeCrts []*v1alpha2.Certificate) {
activeMap := make(map[string]struct{}, len(activeCrts))
for _, crt := range activeCrts {
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt)
if err != nil {
continue
}
activeMap[key] = struct{}{}
}
registeredCertificates.mtx.Lock()
defer registeredCertificates.mtx.Unlock()
var toCleanUp []string
for key := range registeredCertificates.certificates {
if _, found := activeMap[key]; !found {
toCleanUp = append(toCleanUp, key)
}
}
for _, key := range toCleanUp {
cleanUpCertificateByKey(key)
}
}
// metricCleanUpCertificate creates a clean up function which deletes the entry
// (if any) for a certificate in the given metric
func metricCleanUpCertificate(c cleanableMetric) func(string, string) {
return func(name, namespace string) {
c.DeleteLabelValues(name, namespace)
}
}
// metricCleanUpCertificateWith creates a clean up function which deletes the
// entries (if any) for a certificate in the given metric, iterating over the
// additional labels.
// This is used if the metric keys on data in addition to the name and
// namespace.
func metricCleanUpCertificateWith(c cleanableMetric, additionalLabels []string) func(string, string) {
return func(name, namespace string) {
for _, label := range additionalLabels {
c.DeleteLabelValues(name, namespace, label)
}
}
}
// cleanUpCertificateByKey removes metrics which refer to a certificate,
// given the key of the certificate.
func cleanUpCertificateByKey(key string) {
namespace, name, err := cache.SplitMetaNamespaceKey(key) namespace, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil { if err != nil {
m.log.Error(err, "failed to get namespace and name from key")
return return
} }
// apply all the clean up functions // If the certificate is not registered, exit early
for _, f := range cleanUpFunctions { if _, ok := m.registeredCertificates[key]; !ok {
f(name, namespace) return
} }
delete(registeredCertificates.certificates, key) m.certificateExpiryTimeSeconds.DeleteLabelValues(name, namespace)
for _, condition := range readyConditionStatuses {
m.certificateReadyStatus.DeleteLabelValues(name, namespace, string(condition))
}
delete(m.registeredCertificates, key)
} }
func (m *Metrics) IncrementSyncCallCount(controllerName string) { func (m *Metrics) shutdown() {
log := logf.FromContext(m.ctx) m.log.Info("stopping Prometheus metrics server...")
log.V(logf.DebugLevel).Info("incrementing controller sync call count", "controllerName", controllerName)
ControllerSyncCallCount.WithLabelValues(controllerName).Inc() ctx, cancel := context.WithTimeout(context.Background(), prometheusMetricsServerShutdownTimeout)
defer cancel()
if err := m.server.Shutdown(ctx); err != nil {
m.log.Error(err, "prometheus metrics server shutdown failed", err)
return
}
m.log.Info("prometheus metrics server gracefully stopped")
} }

404
pkg/metrics/metrics_test.go
View File

@ -1,5 +1,5 @@
/* /*
Copyright 2019 The Jetstack cert-manager contributors. Copyright 2020 The Jetstack cert-manager contributors.
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
@ -17,7 +17,7 @@ limitations under the License.
package metrics package metrics
import ( import (
"crypto/x509" "context"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -25,124 +25,118 @@ import (
"github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/client_golang/prometheus/testutil"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2" cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1" cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
logtesting "github.com/jetstack/cert-manager/pkg/logs/testing"
"github.com/jetstack/cert-manager/test/unit/gen"
) )
func getReadyConditionStatus(crt *v1alpha2.Certificate) cmmeta.ConditionStatus { const expiryMetadata = `
for _, c := range crt.Status.Conditions {
switch c.Type {
case v1alpha2.CertificateConditionReady:
return c.Status
}
}
return cmmeta.ConditionUnknown
}
func buildCertificate(name, namespace string, condition cmmeta.ConditionStatus) *v1alpha2.Certificate {
return &v1alpha2.Certificate{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Status: v1alpha2.CertificateStatus{
Conditions: []v1alpha2.CertificateCondition{
{
Type: v1alpha2.CertificateConditionReady,
Status: condition,
},
},
},
}
}
func TestUpdateCertificateExpiry(t *testing.T) {
const metadata = `
# HELP certmanager_certificate_expiration_timestamp_seconds The date after which the certificate expires. Expressed as a Unix Epoch Time. # HELP certmanager_certificate_expiration_timestamp_seconds The date after which the certificate expires. Expressed as a Unix Epoch Time.
# TYPE certmanager_certificate_expiration_timestamp_seconds gauge # TYPE certmanager_certificate_expiration_timestamp_seconds gauge
` `
const readyMetadata = `
# HELP certmanager_certificate_ready_status The ready status of the certificate.
# TYPE certmanager_certificate_ready_status gauge
`
func TestCertificateMetrics(t *testing.T) {
type testT struct { type testT struct {
crt *v1alpha2.Certificate crt *cmapi.Certificate
cert *x509.Certificate expectedExpiry, expectedReady string
expected string
} }
tests := map[string]testT{ tests := map[string]testT{
"first": { "certificate with expiry and ready status": {
crt: &v1alpha2.Certificate{ crt: gen.Certificate("test-certificate",
ObjectMeta: metav1.ObjectMeta{ gen.SetCertificateNamespace("test-ns"),
Name: "something", gen.SetCertificateNotAfter(metav1.Time{
Namespace: "default", Time: time.Unix(2208988804, 0),
}, }),
}, gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
cert: &x509.Certificate{ Type: cmapi.CertificateConditionReady,
// fixed expiry time for testing Status: cmmeta.ConditionTrue,
NotAfter: time.Unix(2208988804, 0), }),
}, ),
expected: ` expectedExpiry: `
certmanager_certificate_expiration_timestamp_seconds{name="something",namespace="default"} 2.208988804e+09 certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 2.208988804e+09
`,
expectedReady: `
certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 0
certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 1
certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 0
`,
},
"certificate with no expiry and no status should give an expiry of 0 and Unknown status": {
crt: gen.Certificate("test-certificate",
gen.SetCertificateNamespace("test-ns"),
),
expectedExpiry: `
certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 0
`,
expectedReady: `
certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 0
certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 0
certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 1
`,
},
"certificate with expiry and status False should give an expiry and False status": {
crt: gen.Certificate("test-certificate",
gen.SetCertificateNamespace("test-ns"),
gen.SetCertificateNotAfter(metav1.Time{
Time: time.Unix(100, 0),
}),
gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
Type: cmapi.CertificateConditionReady,
Status: cmmeta.ConditionFalse,
}),
),
expectedExpiry: `
certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 100
`,
expectedReady: `
certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 1
certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 0
certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 0
`,
},
"certificate with expiry and status Unknown should give an expiry and Unknown status": {
crt: gen.Certificate("test-certificate",
gen.SetCertificateNamespace("test-ns"),
gen.SetCertificateNotAfter(metav1.Time{
Time: time.Unix(99999, 0),
}),
gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
Type: cmapi.CertificateConditionReady,
Status: cmmeta.ConditionUnknown,
}),
),
expectedExpiry: `
certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 99999
`,
expectedReady: `
certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 0
certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 0
certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 1
`, `,
}, },
} }
for n, test := range tests { for n, test := range tests {
t.Run(n, func(t *testing.T) { t.Run(n, func(t *testing.T) {
defer cleanUpCertificates(nil) m := New(logtesting.TestLogger{T: t})
m.UpdateCertificate(context.TODO(), test.crt)
updateX509Expiry(test.crt, test.cert) if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds,
strings.NewReader(expiryMetadata+test.expectedExpiry),
if err := testutil.CollectAndCompare(
CertificateExpiryTimeSeconds,
strings.NewReader(metadata+test.expected),
"certmanager_certificate_expiration_timestamp_seconds", "certmanager_certificate_expiration_timestamp_seconds",
); err != nil { ); err != nil {
t.Errorf("unexpected collecting result:\n%s", err) t.Errorf("unexpected collecting result:\n%s", err)
} }
})
}
}
func TestUpdateCertificateReadyStatus(t *testing.T) { if err := testutil.CollectAndCompare(m.certificateReadyStatus,
const metadata = ` strings.NewReader(readyMetadata+test.expectedReady),
# HELP certmanager_certificate_ready_status The ready status of the certificate.
# TYPE certmanager_certificate_ready_status gauge
`
type testT struct {
crt *v1alpha2.Certificate
expected string
}
tests := map[string]testT{
"ready status true is updated correctly": {
crt: buildCertificate("something", "default", cmmeta.ConditionTrue),
expected: `
certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 0
certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 1
certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 0
`,
},
"ready status false is updated correctly": {
crt: buildCertificate("something", "default", cmmeta.ConditionFalse),
expected: `
certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 1
certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 0
certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 0
`,
},
"ready status unknown is updated correctly": {
crt: buildCertificate("something", "default", cmmeta.ConditionUnknown),
expected: `
certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 0
certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 0
certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 1
`,
},
}
for n, test := range tests {
t.Run(n, func(t *testing.T) {
updateCertificateReadyStatus(test.crt, getReadyConditionStatus(test.crt))
if err := testutil.CollectAndCompare(
CertificateReadyStatus,
strings.NewReader(metadata+test.expected),
"certmanager_certificate_ready_status", "certmanager_certificate_ready_status",
); err != nil { ); err != nil {
t.Errorf("unexpected collecting result:\n%s", err) t.Errorf("unexpected collecting result:\n%s", err)
@ -151,118 +145,112 @@ func TestUpdateCertificateReadyStatus(t *testing.T) {
} }
} }
func TestCleanUp(t *testing.T) { func TestCertificateCache(t *testing.T) {
const metadataExpiry = ` m := New(logtesting.TestLogger{T: t})
# HELP certmanager_certificate_expiration_timestamp_seconds The date after which the certificate expires. Expressed as a Unix Epoch Time.
# TYPE certmanager_certificate_expiration_timestamp_seconds gauge
`
const metadataReady = ` crt1 := gen.Certificate("crt1",
# HELP certmanager_certificate_ready_status The ready status of the certificate. gen.SetCertificateUID("uid-1"),
# TYPE certmanager_certificate_ready_status gauge gen.SetCertificateNotAfter(metav1.Time{
` Time: time.Unix(100, 0),
type testT struct { }),
active map[*v1alpha2.Certificate]*x509.Certificate gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
inactive map[*v1alpha2.Certificate]*x509.Certificate Type: cmapi.CertificateConditionReady,
expectedExpiry string Status: cmmeta.ConditionUnknown,
expectedReady string }),
)
crt2 := gen.Certificate("crt2",
gen.SetCertificateUID("uid-2"),
gen.SetCertificateNotAfter(metav1.Time{
Time: time.Unix(200, 0),
}),
gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
Type: cmapi.CertificateConditionReady,
Status: cmmeta.ConditionTrue,
}),
)
crt3 := gen.Certificate("crt3",
gen.SetCertificateUID("uid-3"),
gen.SetCertificateNotAfter(metav1.Time{
Time: time.Unix(300, 0),
}),
gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
Type: cmapi.CertificateConditionReady,
Status: cmmeta.ConditionFalse,
}),
)
// Observe all three Certificate metrics
m.UpdateCertificate(context.TODO(), crt1)
m.UpdateCertificate(context.TODO(), crt2)
m.UpdateCertificate(context.TODO(), crt3)
// Check all three metrics exist
if err := testutil.CollectAndCompare(m.certificateReadyStatus,
strings.NewReader(readyMetadata+`
certmanager_certificate_ready_status{condition="False",name="crt1",namespace="default-unit-test-ns"} 0
certmanager_certificate_ready_status{condition="False",name="crt2",namespace="default-unit-test-ns"} 0
certmanager_certificate_ready_status{condition="False",name="crt3",namespace="default-unit-test-ns"} 1
certmanager_certificate_ready_status{condition="True",name="crt1",namespace="default-unit-test-ns"} 0
certmanager_certificate_ready_status{condition="True",name="crt2",namespace="default-unit-test-ns"} 1
certmanager_certificate_ready_status{condition="True",name="crt3",namespace="default-unit-test-ns"} 0
certmanager_certificate_ready_status{condition="Unknown",name="crt1",namespace="default-unit-test-ns"} 1
certmanager_certificate_ready_status{condition="Unknown",name="crt2",namespace="default-unit-test-ns"} 0
certmanager_certificate_ready_status{condition="Unknown",name="crt3",namespace="default-unit-test-ns"} 0
`),
"certmanager_certificate_ready_status",
); err != nil {
t.Errorf("unexpected collecting result:\n%s", err)
} }
tests := map[string]testT{ if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds,
"inactive certificate metrics cleaned up while active certificate metrics kept": { strings.NewReader(expiryMetadata+`
active: map[*v1alpha2.Certificate]*x509.Certificate{ certmanager_certificate_expiration_timestamp_seconds{name="crt1",namespace="default-unit-test-ns"} 100
buildCertificate("active", "default", cmmeta.ConditionTrue): { certmanager_certificate_expiration_timestamp_seconds{name="crt2",namespace="default-unit-test-ns"} 200
// fixed expiry time for testing certmanager_certificate_expiration_timestamp_seconds{name="crt3",namespace="default-unit-test-ns"} 300
NotAfter: time.Unix(2208988804, 0), `),
}, "certmanager_certificate_expiration_timestamp_seconds",
}, ); err != nil {
inactive: map[*v1alpha2.Certificate]*x509.Certificate{ t.Errorf("unexpected collecting result:\n%s", err)
buildCertificate("inactive", "default", cmmeta.ConditionTrue): {
// fixed expiry time for testing
NotAfter: time.Unix(2208988804, 0),
},
},
expectedExpiry: `
certmanager_certificate_expiration_timestamp_seconds{name="active",namespace="default"} 2.208988804e+09
`,
expectedReady: `
certmanager_certificate_ready_status{condition="False",name="active",namespace="default"} 0
certmanager_certificate_ready_status{condition="True",name="active",namespace="default"} 1
certmanager_certificate_ready_status{condition="Unknown",name="active",namespace="default"} 0
`,
},
"no metrics cleaned up when only active certificate metrics": {
active: map[*v1alpha2.Certificate]*x509.Certificate{
buildCertificate("active", "default", cmmeta.ConditionTrue): {
// fixed expiry time for testing
NotAfter: time.Unix(2208988804, 0),
},
buildCertificate("also-active", "default", cmmeta.ConditionTrue): {
// fixed expiry time for testing
NotAfter: time.Unix(2208988804, 0),
},
},
inactive: map[*v1alpha2.Certificate]*x509.Certificate{},
expectedExpiry: `
certmanager_certificate_expiration_timestamp_seconds{name="active",namespace="default"} 2.208988804e+09
certmanager_certificate_expiration_timestamp_seconds{name="also-active",namespace="default"} 2.208988804e+09
`,
expectedReady: `
certmanager_certificate_ready_status{condition="False",name="active",namespace="default"} 0
certmanager_certificate_ready_status{condition="False",name="also-active",namespace="default"} 0
certmanager_certificate_ready_status{condition="True",name="active",namespace="default"} 1
certmanager_certificate_ready_status{condition="True",name="also-active",namespace="default"} 1
certmanager_certificate_ready_status{condition="Unknown",name="active",namespace="default"} 0
certmanager_certificate_ready_status{condition="Unknown",name="also-active",namespace="default"} 0
`,
},
"all metrics cleaned up when only inactive certificate metrics": {
active: map[*v1alpha2.Certificate]*x509.Certificate{},
inactive: map[*v1alpha2.Certificate]*x509.Certificate{
buildCertificate("inactive", "default", cmmeta.ConditionTrue): {
// fixed expiry time for testing
NotAfter: time.Unix(2208988804, 0),
},
buildCertificate("also-inactive", "default", cmmeta.ConditionTrue): {
// fixed expiry time for testing
NotAfter: time.Unix(2208988804, 0),
},
},
expectedExpiry: "",
expectedReady: "",
},
} }
for n, test := range tests {
t.Run(n, func(t *testing.T) {
defer cleanUpCertificates(nil)
var activeCrts []*v1alpha2.Certificate // Remove second certificate and check not exists
for crt, cert := range test.active { m.RemoveCertificate("default-unit-test-ns/crt2")
updateX509Expiry(crt, cert) if err := testutil.CollectAndCompare(m.certificateReadyStatus,
updateCertificateReadyStatus(crt, getReadyConditionStatus(crt)) strings.NewReader(readyMetadata+`
activeCrts = append(activeCrts, crt) certmanager_certificate_ready_status{condition="False",name="crt1",namespace="default-unit-test-ns"} 0
} certmanager_certificate_ready_status{condition="False",name="crt3",namespace="default-unit-test-ns"} 1
for crt, cert := range test.inactive { certmanager_certificate_ready_status{condition="True",name="crt1",namespace="default-unit-test-ns"} 0
updateCertificateReadyStatus(crt, getReadyConditionStatus(crt)) certmanager_certificate_ready_status{condition="True",name="crt3",namespace="default-unit-test-ns"} 0
updateX509Expiry(crt, cert) certmanager_certificate_ready_status{condition="Unknown",name="crt1",namespace="default-unit-test-ns"} 1
} certmanager_certificate_ready_status{condition="Unknown",name="crt3",namespace="default-unit-test-ns"} 0
`),
"certmanager_certificate_ready_status",
); err != nil {
t.Errorf("unexpected collecting result:\n%s", err)
}
if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds,
strings.NewReader(expiryMetadata+`
certmanager_certificate_expiration_timestamp_seconds{name="crt1",namespace="default-unit-test-ns"} 100
certmanager_certificate_expiration_timestamp_seconds{name="crt3",namespace="default-unit-test-ns"} 300
`),
"certmanager_certificate_expiration_timestamp_seconds",
); err != nil {
t.Errorf("unexpected collecting result:\n%s", err)
}
cleanUpCertificates(activeCrts) // Remove all Certificates (even is already removed) and observe no Certificates
m.RemoveCertificate("default-unit-test-ns/crt1")
if err := testutil.CollectAndCompare( m.RemoveCertificate("default-unit-test-ns/crt2")
CertificateExpiryTimeSeconds, m.RemoveCertificate("default-unit-test-ns/crt3")
strings.NewReader(metadataExpiry+test.expectedExpiry), if err := testutil.CollectAndCompare(m.certificateReadyStatus,
"certmanager_certificate_expiration_timestamp_seconds", strings.NewReader(readyMetadata),
); err != nil { "certmanager_certificate_ready_status",
t.Errorf("unexpected collecting result:\n%s", err) ); err != nil {
} t.Errorf("unexpected collecting result:\n%s", err)
}
if err := testutil.CollectAndCompare( if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds,
CertificateReadyStatus, strings.NewReader(expiryMetadata),
strings.NewReader(metadataReady+test.expectedReady), "certmanager_certificate_expiration_timestamp_seconds",
"certmanager_certificate_ready_status", ); err != nil {
); err != nil { t.Errorf("unexpected collecting result:\n%s", err)
t.Errorf("unexpected collecting result:\n%s", err)
}
})
} }
} }