Updates metrics package to be better consumable
Signed-off-by: JoshVanL <vleeuwenjoshua@gmail.com>
This commit is contained in:
JoshVanL
parent
6a6f3dbf7a
commit
9e98d7b948
@ -8,17 +8,11 @@ go_library(
|
||||
deps = [
|
||||
"//pkg/apis/certmanager/v1alpha2:go_default_library",
|
||||
"//pkg/apis/meta/v1:go_default_library",
|
||||
"//pkg/client/listers/certmanager/v1alpha2:go_default_library",
|
||||
"//pkg/logs:go_default_library",
|
||||
"//pkg/util/errors:go_default_library",
|
||||
"//pkg/util/kube:go_default_library",
|
||||
"@com_github_go_logr_logr//:go_default_library",
|
||||
"@com_github_gorilla_mux//:go_default_library",
|
||||
"@com_github_prometheus_client_golang//prometheus:go_default_library",
|
||||
"@com_github_prometheus_client_golang//prometheus/promhttp:go_default_library",
|
||||
"@io_k8s_apimachinery//pkg/api/errors:go_default_library",
|
||||
"@io_k8s_apimachinery//pkg/labels:go_default_library",
|
||||
"@io_k8s_apimachinery//pkg/util/wait:go_default_library",
|
||||
"@io_k8s_client_go//listers/core/v1:go_default_library",
|
||||
"@io_k8s_client_go//tools/cache:go_default_library",
|
||||
],
|
||||
)
|
||||
@ -44,6 +38,8 @@ go_test(
|
||||
deps = [
|
||||
"//pkg/apis/certmanager/v1alpha2:go_default_library",
|
||||
"//pkg/apis/meta/v1:go_default_library",
|
||||
"//pkg/logs/testing:go_default_library",
|
||||
"//test/unit/gen:go_default_library",
|
||||
"@com_github_prometheus_client_golang//prometheus/testutil:go_default_library",
|
||||
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
|
||||
],
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2019 The Jetstack cert-manager contributors.
|
||||
Copyright 2020 The Jetstack cert-manager contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
@ -18,370 +18,280 @@ limitations under the License.
|
||||
// cert-manager exposes the following metrics:
|
||||
// certificate_expiration_timestamp_seconds{name, namespace}
|
||||
// certificate_ready_status{name, namespace, condition}
|
||||
// acme_client_request_count{"scheme", "host", "path", "method", "status"}
|
||||
// acme_client_request_duration_seconds{"scheme", "host", "path", "method", "status"}
|
||||
// controller_sync_call_count{"controller"}
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
corelisters "k8s.io/client-go/listers/core/v1"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
|
||||
"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
|
||||
cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
|
||||
cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
|
||||
cmlisters "github.com/jetstack/cert-manager/pkg/client/listers/certmanager/v1alpha2"
|
||||
logf "github.com/jetstack/cert-manager/pkg/logs"
|
||||
"github.com/jetstack/cert-manager/pkg/util/errors"
|
||||
"github.com/jetstack/cert-manager/pkg/util/kube"
|
||||
)
|
||||
|
||||
const (
|
||||
// Namespace is the namespace for cert-manager metric names
|
||||
namespace = "certmanager"
|
||||
prometheusMetricsServerAddress = "0.0.0.0:9402"
|
||||
prometheusMetricsServerAddress = "127.0.0.1:9402"
|
||||
prometheusMetricsServerShutdownTimeout = 5 * time.Second
|
||||
prometheusMetricsServerReadTimeout = 8 * time.Second
|
||||
prometheusMetricsServerWriteTimeout = 8 * time.Second
|
||||
prometheusMetricsServerMaxHeaderBytes = 1 << 20 // 1 MiB
|
||||
)
|
||||
|
||||
var readyConditionStatuses = [...]string{string(cmmeta.ConditionTrue), string(cmmeta.ConditionFalse), string(cmmeta.ConditionUnknown)}
|
||||
|
||||
// Default set of metrics
|
||||
var Default = New(logf.NewContext(context.Background(), logf.Log.WithName("metrics")))
|
||||
|
||||
var CertificateExpiryTimeSeconds = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "certificate_expiration_timestamp_seconds",
|
||||
Help: "The date after which the certificate expires. Expressed as a Unix Epoch Time.",
|
||||
},
|
||||
[]string{"name", "namespace"},
|
||||
)
|
||||
|
||||
var CertificateReadyStatus = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "certificate_ready_status",
|
||||
Help: "The ready status of the certificate.",
|
||||
},
|
||||
[]string{"name", "namespace", "condition"},
|
||||
)
|
||||
|
||||
// ACMEClientRequestCount is a Prometheus summary to collect the number of
|
||||
// requests made to each endpoint with the ACME client.
|
||||
var ACMEClientRequestCount = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "acme_client_request_count",
|
||||
Help: "The number of requests made by the ACME client.",
|
||||
Subsystem: "http",
|
||||
},
|
||||
[]string{"scheme", "host", "path", "method", "status"},
|
||||
)
|
||||
|
||||
// ACMEClientRequestDurationSeconds is a Prometheus summary to collect request
|
||||
// times for the ACME client.
|
||||
var ACMEClientRequestDurationSeconds = prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: namespace,
|
||||
Name: "acme_client_request_duration_seconds",
|
||||
Help: "The HTTP request latencies in seconds for the ACME client.",
|
||||
Subsystem: "http",
|
||||
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
|
||||
},
|
||||
[]string{"scheme", "host", "path", "method", "status"},
|
||||
)
|
||||
|
||||
var ControllerSyncCallCount = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "controller_sync_call_count",
|
||||
Help: "The number of sync() calls made by a controller.",
|
||||
},
|
||||
[]string{"controller"},
|
||||
)
|
||||
|
||||
// registeredCertificates holds the set of all certificates which are currently
|
||||
// registered by Prometheus
|
||||
var registeredCertificates = &struct {
|
||||
certificates map[string]struct{}
|
||||
mtx sync.Mutex
|
||||
}{
|
||||
certificates: make(map[string]struct{}),
|
||||
}
|
||||
|
||||
// cleanUpFunctions are functions called to clean up metrics which refer to
|
||||
// deleted certificates, inputs are name and namespace of the certificate
|
||||
var cleanUpFunctions = []func(string, string){
|
||||
metricCleanUpCertificate(CertificateExpiryTimeSeconds),
|
||||
metricCleanUpCertificateWith(CertificateReadyStatus, readyConditionStatuses[:]),
|
||||
}
|
||||
|
||||
type cleanableMetric interface {
|
||||
DeleteLabelValues(...string) bool
|
||||
}
|
||||
|
||||
// Metrics is designed to be a shared object for updating the metrics exposed
|
||||
// by cert-manager
|
||||
type Metrics struct {
|
||||
ctx context.Context
|
||||
http.Server
|
||||
activeCertificates cmlisters.CertificateLister
|
||||
log logr.Logger
|
||||
registry *prometheus.Registry
|
||||
server *http.Server
|
||||
|
||||
// TODO (@dippynark): switch this to use an interface to make it testable
|
||||
registry *prometheus.Registry
|
||||
CertificateExpiryTimeSeconds *prometheus.GaugeVec
|
||||
CertificateReadyStatus *prometheus.GaugeVec
|
||||
ACMEClientRequestDurationSeconds *prometheus.SummaryVec
|
||||
ACMEClientRequestCount *prometheus.CounterVec
|
||||
ControllerSyncCallCount *prometheus.CounterVec
|
||||
mux sync.Mutex
|
||||
|
||||
certificateExpiryTimeSeconds *prometheus.GaugeVec
|
||||
certificateReadyStatus *prometheus.GaugeVec
|
||||
acmeClientRequestDurationSeconds *prometheus.SummaryVec
|
||||
acmeClientRequestCount *prometheus.CounterVec
|
||||
controllerSyncCallCount *prometheus.CounterVec
|
||||
|
||||
registeredCertificates map[string]struct{}
|
||||
}
|
||||
|
||||
func New(ctx context.Context) *Metrics {
|
||||
var readyConditionStatuses = [...]cmmeta.ConditionStatus{cmmeta.ConditionTrue, cmmeta.ConditionFalse, cmmeta.ConditionUnknown}
|
||||
|
||||
func New(log logr.Logger) *Metrics {
|
||||
var (
|
||||
certificateExpiryTimeSeconds = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "certificate_expiration_timestamp_seconds",
|
||||
Help: "The date after which the certificate expires. Expressed as a Unix Epoch Time.",
|
||||
},
|
||||
[]string{"name", "namespace"},
|
||||
)
|
||||
|
||||
certificateReadyStatus = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "certificate_ready_status",
|
||||
Help: "The ready status of the certificate.",
|
||||
},
|
||||
[]string{"name", "namespace", "condition"},
|
||||
)
|
||||
|
||||
// acmeClientRequestCount is a Prometheus summary to collect the number of
|
||||
// requests made to each endpoint with the ACME client.
|
||||
acmeClientRequestCount = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "acme_client_request_count",
|
||||
Help: "The number of requests made by the ACME client.",
|
||||
Subsystem: "http",
|
||||
},
|
||||
[]string{"scheme", "host", "path", "method", "status"},
|
||||
)
|
||||
|
||||
// acmeClientRequestDurationSeconds is a Prometheus summary to collect request
|
||||
// times for the ACME client.
|
||||
acmeClientRequestDurationSeconds = prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: namespace,
|
||||
Name: "acme_client_request_duration_seconds",
|
||||
Help: "The HTTP request latencies in seconds for the ACME client.",
|
||||
Subsystem: "http",
|
||||
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
|
||||
},
|
||||
[]string{"scheme", "host", "path", "method", "status"},
|
||||
)
|
||||
|
||||
controllerSyncCallCount = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "controller_sync_call_count",
|
||||
Help: "The number of sync() calls made by a controller.",
|
||||
},
|
||||
[]string{"controller"},
|
||||
)
|
||||
)
|
||||
|
||||
router := mux.NewRouter()
|
||||
|
||||
// Create server and register prometheus metrics handler
|
||||
s := &Metrics{
|
||||
ctx: ctx,
|
||||
Server: http.Server{
|
||||
// Create server and register Prometheus metrics handler
|
||||
m := &Metrics{
|
||||
log: log.WithName("metrics"),
|
||||
registry: prometheus.NewRegistry(),
|
||||
server: &http.Server{
|
||||
Addr: prometheusMetricsServerAddress,
|
||||
ReadTimeout: prometheusMetricsServerReadTimeout,
|
||||
WriteTimeout: prometheusMetricsServerWriteTimeout,
|
||||
MaxHeaderBytes: prometheusMetricsServerMaxHeaderBytes,
|
||||
Handler: router,
|
||||
},
|
||||
activeCertificates: nil,
|
||||
registry: prometheus.NewRegistry(),
|
||||
CertificateExpiryTimeSeconds: CertificateExpiryTimeSeconds,
|
||||
CertificateReadyStatus: CertificateReadyStatus,
|
||||
ACMEClientRequestDurationSeconds: ACMEClientRequestDurationSeconds,
|
||||
ACMEClientRequestCount: ACMEClientRequestCount,
|
||||
ControllerSyncCallCount: ControllerSyncCallCount,
|
||||
|
||||
registeredCertificates: make(map[string]struct{}),
|
||||
|
||||
certificateExpiryTimeSeconds: certificateExpiryTimeSeconds,
|
||||
certificateReadyStatus: certificateReadyStatus,
|
||||
acmeClientRequestCount: acmeClientRequestCount,
|
||||
acmeClientRequestDurationSeconds: acmeClientRequestDurationSeconds,
|
||||
controllerSyncCallCount: controllerSyncCallCount,
|
||||
}
|
||||
|
||||
router.Handle("/metrics", promhttp.HandlerFor(s.registry, promhttp.HandlerOpts{}))
|
||||
router.Handle("/metrics", promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{}))
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func (m *Metrics) waitShutdown(stopCh <-chan struct{}) {
|
||||
log := logf.FromContext(m.ctx)
|
||||
<-stopCh
|
||||
log.Info("stopping Prometheus metrics server...")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), prometheusMetricsServerShutdownTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := m.Shutdown(ctx); err != nil {
|
||||
log.Error(err, "prometheus metrics server shutdown failed", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("prometheus metrics server gracefully stopped")
|
||||
return m
|
||||
}
|
||||
|
||||
// Start will register the Prometheu metrics, and start the Prometheus server
|
||||
func (m *Metrics) Start(stopCh <-chan struct{}) {
|
||||
log := logf.FromContext(m.ctx)
|
||||
|
||||
m.registry.MustRegister(m.CertificateExpiryTimeSeconds)
|
||||
m.registry.MustRegister(m.CertificateReadyStatus)
|
||||
m.registry.MustRegister(m.ACMEClientRequestDurationSeconds)
|
||||
m.registry.MustRegister(m.ACMEClientRequestCount)
|
||||
m.registry.MustRegister(m.ControllerSyncCallCount)
|
||||
m.registry.MustRegister(m.certificateExpiryTimeSeconds)
|
||||
m.registry.MustRegister(m.certificateReadyStatus)
|
||||
m.registry.MustRegister(m.acmeClientRequestDurationSeconds)
|
||||
m.registry.MustRegister(m.acmeClientRequestCount)
|
||||
m.registry.MustRegister(m.controllerSyncCallCount)
|
||||
|
||||
go func() {
|
||||
log := log.WithValues("address", m.Addr)
|
||||
log := m.log.WithValues("address", m.server.Addr)
|
||||
log.Info("listening for connections on")
|
||||
if err := m.ListenAndServe(); err != nil {
|
||||
if err := m.server.ListenAndServe(); err != nil {
|
||||
log.Error(err, "error running prometheus metrics server")
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("prometheus metrics server exited")
|
||||
|
||||
}()
|
||||
|
||||
// clean up metrics referring to deleted resources every minute
|
||||
go wait.Until(func() { m.cleanUp() }, time.Minute, stopCh)
|
||||
|
||||
m.waitShutdown(stopCh)
|
||||
<-stopCh
|
||||
m.shutdown()
|
||||
}
|
||||
|
||||
// UpdateCertificateExpiry updates the expiry time of a certificate
|
||||
func (m *Metrics) UpdateCertificateExpiry(crt *v1alpha2.Certificate, secretLister corelisters.SecretLister) {
|
||||
log := logf.FromContext(m.ctx)
|
||||
log = logf.WithResource(log, crt)
|
||||
log = logf.WithRelatedResourceName(log, crt.Spec.SecretName, crt.Namespace, "Secret")
|
||||
// ObserveACMERequestDuration increases bucket counters for that ACME client duration.
|
||||
func (m *Metrics) ObserveACMERequestDuration(duration time.Duration, labels ...string) {
|
||||
m.acmeClientRequestDurationSeconds.WithLabelValues(labels...).Observe(duration.Seconds())
|
||||
}
|
||||
|
||||
log.V(logf.DebugLevel).Info("attempting to retrieve secret for certificate")
|
||||
// grab existing certificate
|
||||
cert, err := kube.SecretTLSCert(m.ctx, secretLister, crt.Namespace, crt.Spec.SecretName)
|
||||
// IncrementACMERequestCount increases the acme client request counter.
|
||||
func (m *Metrics) IncrementACMERequestCount(labels ...string) {
|
||||
m.acmeClientRequestCount.WithLabelValues(labels...).Inc()
|
||||
}
|
||||
|
||||
// IncrementSyncCallCount will increase the sync counter for that controller.
|
||||
func (m *Metrics) IncrementSyncCallCount(controllerName string) {
|
||||
m.controllerSyncCallCount.WithLabelValues(controllerName).Inc()
|
||||
}
|
||||
|
||||
// UpdateCertificate will update that Certificate metric with expiry and Ready
|
||||
// condition.
|
||||
func (m *Metrics) UpdateCertificate(ctx context.Context, crt *cmapi.Certificate) {
|
||||
key, err := cache.MetaNamespaceKeyFunc(crt)
|
||||
if err != nil {
|
||||
if !apierrors.IsNotFound(err) && !errors.IsInvalidData(err) {
|
||||
log.Error(err, "error reading secret for certificate")
|
||||
}
|
||||
log := logf.WithRelatedResource(m.log, crt)
|
||||
log.Error(err, "failed to get key from certificate object")
|
||||
return
|
||||
}
|
||||
|
||||
updateX509Expiry(crt, cert)
|
||||
m.updateCertificateStatus(key, crt)
|
||||
m.updateCertificateExpiry(ctx, key, crt)
|
||||
}
|
||||
|
||||
func updateX509Expiry(crt *v1alpha2.Certificate, cert *x509.Certificate) {
|
||||
expiryTime := cert.NotAfter
|
||||
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt)
|
||||
if err != nil {
|
||||
return
|
||||
// updateCertificateExpiry updates the expiry time of a certificate
|
||||
func (m *Metrics) updateCertificateExpiry(ctx context.Context, key string, crt *cmapi.Certificate) {
|
||||
expiryTime := 0.0
|
||||
|
||||
if crt.Status.NotAfter != nil {
|
||||
expiryTime = float64(crt.Status.NotAfter.Unix())
|
||||
}
|
||||
|
||||
registeredCertificates.mtx.Lock()
|
||||
defer registeredCertificates.mtx.Unlock()
|
||||
// set certificate expiry time
|
||||
CertificateExpiryTimeSeconds.With(prometheus.Labels{
|
||||
m.certificateExpiryTimeSeconds.With(prometheus.Labels{
|
||||
"name": crt.Name,
|
||||
"namespace": crt.Namespace}).Set(float64(expiryTime.Unix()))
|
||||
registerCertificateKey(key)
|
||||
"namespace": crt.Namespace}).Set(expiryTime)
|
||||
|
||||
m.mux.Lock()
|
||||
m.registeredCertificates[key] = struct{}{}
|
||||
m.mux.Unlock()
|
||||
}
|
||||
|
||||
func (m *Metrics) UpdateCertificateStatus(crt *v1alpha2.Certificate) {
|
||||
log := logf.FromContext(m.ctx)
|
||||
log = logf.WithResource(log, crt)
|
||||
// updateCertificateStatus will update the metric for that Certificate
|
||||
func (m *Metrics) updateCertificateStatus(key string, crt *cmapi.Certificate) {
|
||||
defer func() {
|
||||
m.mux.Lock()
|
||||
m.registeredCertificates[key] = struct{}{}
|
||||
m.mux.Unlock()
|
||||
}()
|
||||
|
||||
log.V(logf.DebugLevel).Info("attempting to retrieve ready status for certificate")
|
||||
for _, c := range crt.Status.Conditions {
|
||||
switch c.Type {
|
||||
case v1alpha2.CertificateConditionReady:
|
||||
updateCertificateReadyStatus(crt, c.Status)
|
||||
if c.Type == cmapi.CertificateConditionReady {
|
||||
m.updateCertificateReadyStatus(crt, c.Status)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// If no status condition set yet, set to Unknown
|
||||
m.updateCertificateReadyStatus(crt, cmmeta.ConditionUnknown)
|
||||
}
|
||||
|
||||
func updateCertificateReadyStatus(crt *v1alpha2.Certificate, current cmmeta.ConditionStatus) {
|
||||
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
registeredCertificates.mtx.Lock()
|
||||
defer registeredCertificates.mtx.Unlock()
|
||||
func (m *Metrics) updateCertificateReadyStatus(crt *cmapi.Certificate, current cmmeta.ConditionStatus) {
|
||||
for _, condition := range readyConditionStatuses {
|
||||
value := 0.0
|
||||
if string(current) == condition {
|
||||
|
||||
if current == condition {
|
||||
value = 1.0
|
||||
}
|
||||
CertificateReadyStatus.With(prometheus.Labels{
|
||||
|
||||
m.certificateReadyStatus.With(prometheus.Labels{
|
||||
"name": crt.Name,
|
||||
"namespace": crt.Namespace,
|
||||
"condition": string(condition),
|
||||
}).Set(value)
|
||||
}
|
||||
registerCertificateKey(key)
|
||||
}
|
||||
|
||||
// registerCertificateKey adds an entry in registeredCertificates to track
|
||||
// which certificates have metrics stored in prometheus, allowing for easier
|
||||
// clean-up.
|
||||
// You MUST lock the mutex before calling this function, this ensures no other
|
||||
// function is cleaning up while we are registering a certificate
|
||||
func registerCertificateKey(key string) {
|
||||
registeredCertificates.certificates[key] = struct{}{}
|
||||
}
|
||||
// RemoveCertificate will delete the Certificate metrics from continuing to be
|
||||
// exposed.
|
||||
func (m *Metrics) RemoveCertificate(key string) {
|
||||
m.mux.Lock()
|
||||
defer m.mux.Unlock()
|
||||
|
||||
func (m *Metrics) SetActiveCertificates(cl cmlisters.CertificateLister) {
|
||||
m.activeCertificates = cl
|
||||
}
|
||||
|
||||
// cleanUp removes any metrics which reference resources which no longer exist
|
||||
func (m *Metrics) cleanUp() {
|
||||
log := logf.FromContext(m.ctx)
|
||||
log.V(logf.DebugLevel).Info("attempting to clean up metrics for recently deleted certificates")
|
||||
|
||||
if m.activeCertificates == nil {
|
||||
log.V(logf.DebugLevel).Info("active certificates is still uninitialized")
|
||||
return
|
||||
}
|
||||
|
||||
activeCrts, err := m.activeCertificates.List(labels.Everything())
|
||||
if err != nil {
|
||||
log.Error(err, "error retrieving active certificates")
|
||||
return
|
||||
}
|
||||
|
||||
cleanUpCertificates(activeCrts)
|
||||
}
|
||||
|
||||
// cleanUpCertificates removes metrics for recently deleted certificates
|
||||
func cleanUpCertificates(activeCrts []*v1alpha2.Certificate) {
|
||||
activeMap := make(map[string]struct{}, len(activeCrts))
|
||||
for _, crt := range activeCrts {
|
||||
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
activeMap[key] = struct{}{}
|
||||
}
|
||||
|
||||
registeredCertificates.mtx.Lock()
|
||||
defer registeredCertificates.mtx.Unlock()
|
||||
var toCleanUp []string
|
||||
for key := range registeredCertificates.certificates {
|
||||
if _, found := activeMap[key]; !found {
|
||||
toCleanUp = append(toCleanUp, key)
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range toCleanUp {
|
||||
cleanUpCertificateByKey(key)
|
||||
}
|
||||
}
|
||||
|
||||
// metricCleanUpCertificate creates a clean up function which deletes the entry
|
||||
// (if any) for a certificate in the given metric
|
||||
func metricCleanUpCertificate(c cleanableMetric) func(string, string) {
|
||||
return func(name, namespace string) {
|
||||
c.DeleteLabelValues(name, namespace)
|
||||
}
|
||||
}
|
||||
|
||||
// metricCleanUpCertificateWith creates a clean up function which deletes the
|
||||
// entries (if any) for a certificate in the given metric, iterating over the
|
||||
// additional labels.
|
||||
// This is used if the metric keys on data in addition to the name and
|
||||
// namespace.
|
||||
func metricCleanUpCertificateWith(c cleanableMetric, additionalLabels []string) func(string, string) {
|
||||
return func(name, namespace string) {
|
||||
for _, label := range additionalLabels {
|
||||
c.DeleteLabelValues(name, namespace, label)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanUpCertificateByKey removes metrics which refer to a certificate,
|
||||
// given the key of the certificate.
|
||||
func cleanUpCertificateByKey(key string) {
|
||||
namespace, name, err := cache.SplitMetaNamespaceKey(key)
|
||||
if err != nil {
|
||||
m.log.Error(err, "failed to get namespace and name from key")
|
||||
return
|
||||
}
|
||||
|
||||
// apply all the clean up functions
|
||||
for _, f := range cleanUpFunctions {
|
||||
f(name, namespace)
|
||||
// If the certificate is not registered, exit early
|
||||
if _, ok := m.registeredCertificates[key]; !ok {
|
||||
return
|
||||
}
|
||||
|
||||
delete(registeredCertificates.certificates, key)
|
||||
m.certificateExpiryTimeSeconds.DeleteLabelValues(name, namespace)
|
||||
for _, condition := range readyConditionStatuses {
|
||||
m.certificateReadyStatus.DeleteLabelValues(name, namespace, string(condition))
|
||||
}
|
||||
|
||||
delete(m.registeredCertificates, key)
|
||||
}
|
||||
|
||||
func (m *Metrics) IncrementSyncCallCount(controllerName string) {
|
||||
log := logf.FromContext(m.ctx)
|
||||
log.V(logf.DebugLevel).Info("incrementing controller sync call count", "controllerName", controllerName)
|
||||
ControllerSyncCallCount.WithLabelValues(controllerName).Inc()
|
||||
func (m *Metrics) shutdown() {
|
||||
m.log.Info("stopping Prometheus metrics server...")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), prometheusMetricsServerShutdownTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := m.server.Shutdown(ctx); err != nil {
|
||||
m.log.Error(err, "prometheus metrics server shutdown failed", err)
|
||||
return
|
||||
}
|
||||
|
||||
m.log.Info("prometheus metrics server gracefully stopped")
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2019 The Jetstack cert-manager contributors.
|
||||
Copyright 2020 The Jetstack cert-manager contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
@ -17,7 +17,7 @@ limitations under the License.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@ -25,124 +25,118 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
|
||||
cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
|
||||
cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
|
||||
logtesting "github.com/jetstack/cert-manager/pkg/logs/testing"
|
||||
"github.com/jetstack/cert-manager/test/unit/gen"
|
||||
)
|
||||
|
||||
func getReadyConditionStatus(crt *v1alpha2.Certificate) cmmeta.ConditionStatus {
|
||||
for _, c := range crt.Status.Conditions {
|
||||
switch c.Type {
|
||||
case v1alpha2.CertificateConditionReady:
|
||||
return c.Status
|
||||
}
|
||||
}
|
||||
return cmmeta.ConditionUnknown
|
||||
}
|
||||
|
||||
func buildCertificate(name, namespace string, condition cmmeta.ConditionStatus) *v1alpha2.Certificate {
|
||||
return &v1alpha2.Certificate{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Status: v1alpha2.CertificateStatus{
|
||||
Conditions: []v1alpha2.CertificateCondition{
|
||||
{
|
||||
Type: v1alpha2.CertificateConditionReady,
|
||||
Status: condition,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateCertificateExpiry(t *testing.T) {
|
||||
const metadata = `
|
||||
const expiryMetadata = `
|
||||
# HELP certmanager_certificate_expiration_timestamp_seconds The date after which the certificate expires. Expressed as a Unix Epoch Time.
|
||||
# TYPE certmanager_certificate_expiration_timestamp_seconds gauge
|
||||
`
|
||||
|
||||
const readyMetadata = `
|
||||
# HELP certmanager_certificate_ready_status The ready status of the certificate.
|
||||
# TYPE certmanager_certificate_ready_status gauge
|
||||
`
|
||||
|
||||
func TestCertificateMetrics(t *testing.T) {
|
||||
type testT struct {
|
||||
crt *v1alpha2.Certificate
|
||||
cert *x509.Certificate
|
||||
expected string
|
||||
crt *cmapi.Certificate
|
||||
expectedExpiry, expectedReady string
|
||||
}
|
||||
tests := map[string]testT{
|
||||
"first": {
|
||||
crt: &v1alpha2.Certificate{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "something",
|
||||
Namespace: "default",
|
||||
},
|
||||
},
|
||||
cert: &x509.Certificate{
|
||||
// fixed expiry time for testing
|
||||
NotAfter: time.Unix(2208988804, 0),
|
||||
},
|
||||
expected: `
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="something",namespace="default"} 2.208988804e+09
|
||||
"certificate with expiry and ready status": {
|
||||
crt: gen.Certificate("test-certificate",
|
||||
gen.SetCertificateNamespace("test-ns"),
|
||||
gen.SetCertificateNotAfter(metav1.Time{
|
||||
Time: time.Unix(2208988804, 0),
|
||||
}),
|
||||
gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
|
||||
Type: cmapi.CertificateConditionReady,
|
||||
Status: cmmeta.ConditionTrue,
|
||||
}),
|
||||
),
|
||||
expectedExpiry: `
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 2.208988804e+09
|
||||
`,
|
||||
expectedReady: `
|
||||
certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 1
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 0
|
||||
`,
|
||||
},
|
||||
|
||||
"certificate with no expiry and no status should give an expiry of 0 and Unknown status": {
|
||||
crt: gen.Certificate("test-certificate",
|
||||
gen.SetCertificateNamespace("test-ns"),
|
||||
),
|
||||
expectedExpiry: `
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 0
|
||||
`,
|
||||
expectedReady: `
|
||||
certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 1
|
||||
`,
|
||||
},
|
||||
|
||||
"certificate with expiry and status False should give an expiry and False status": {
|
||||
crt: gen.Certificate("test-certificate",
|
||||
gen.SetCertificateNamespace("test-ns"),
|
||||
gen.SetCertificateNotAfter(metav1.Time{
|
||||
Time: time.Unix(100, 0),
|
||||
}),
|
||||
gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
|
||||
Type: cmapi.CertificateConditionReady,
|
||||
Status: cmmeta.ConditionFalse,
|
||||
}),
|
||||
),
|
||||
expectedExpiry: `
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 100
|
||||
`,
|
||||
expectedReady: `
|
||||
certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 1
|
||||
certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 0
|
||||
`,
|
||||
},
|
||||
"certificate with expiry and status Unknown should give an expiry and Unknown status": {
|
||||
crt: gen.Certificate("test-certificate",
|
||||
gen.SetCertificateNamespace("test-ns"),
|
||||
gen.SetCertificateNotAfter(metav1.Time{
|
||||
Time: time.Unix(99999, 0),
|
||||
}),
|
||||
gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
|
||||
Type: cmapi.CertificateConditionReady,
|
||||
Status: cmmeta.ConditionUnknown,
|
||||
}),
|
||||
),
|
||||
expectedExpiry: `
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="test-certificate",namespace="test-ns"} 99999
|
||||
`,
|
||||
expectedReady: `
|
||||
certmanager_certificate_ready_status{condition="False",name="test-certificate",namespace="test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="True",name="test-certificate",namespace="test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="test-certificate",namespace="test-ns"} 1
|
||||
`,
|
||||
},
|
||||
}
|
||||
for n, test := range tests {
|
||||
t.Run(n, func(t *testing.T) {
|
||||
defer cleanUpCertificates(nil)
|
||||
m := New(logtesting.TestLogger{T: t})
|
||||
m.UpdateCertificate(context.TODO(), test.crt)
|
||||
|
||||
updateX509Expiry(test.crt, test.cert)
|
||||
|
||||
if err := testutil.CollectAndCompare(
|
||||
CertificateExpiryTimeSeconds,
|
||||
strings.NewReader(metadata+test.expected),
|
||||
if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds,
|
||||
strings.NewReader(expiryMetadata+test.expectedExpiry),
|
||||
"certmanager_certificate_expiration_timestamp_seconds",
|
||||
); err != nil {
|
||||
t.Errorf("unexpected collecting result:\n%s", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateCertificateReadyStatus(t *testing.T) {
|
||||
const metadata = `
|
||||
# HELP certmanager_certificate_ready_status The ready status of the certificate.
|
||||
# TYPE certmanager_certificate_ready_status gauge
|
||||
`
|
||||
|
||||
type testT struct {
|
||||
crt *v1alpha2.Certificate
|
||||
expected string
|
||||
}
|
||||
tests := map[string]testT{
|
||||
"ready status true is updated correctly": {
|
||||
crt: buildCertificate("something", "default", cmmeta.ConditionTrue),
|
||||
expected: `
|
||||
certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 0
|
||||
certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 1
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 0
|
||||
`,
|
||||
},
|
||||
"ready status false is updated correctly": {
|
||||
crt: buildCertificate("something", "default", cmmeta.ConditionFalse),
|
||||
expected: `
|
||||
certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 1
|
||||
certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 0
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 0
|
||||
`,
|
||||
},
|
||||
"ready status unknown is updated correctly": {
|
||||
crt: buildCertificate("something", "default", cmmeta.ConditionUnknown),
|
||||
expected: `
|
||||
certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 0
|
||||
certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 0
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 1
|
||||
`,
|
||||
},
|
||||
}
|
||||
for n, test := range tests {
|
||||
t.Run(n, func(t *testing.T) {
|
||||
updateCertificateReadyStatus(test.crt, getReadyConditionStatus(test.crt))
|
||||
|
||||
if err := testutil.CollectAndCompare(
|
||||
CertificateReadyStatus,
|
||||
strings.NewReader(metadata+test.expected),
|
||||
if err := testutil.CollectAndCompare(m.certificateReadyStatus,
|
||||
strings.NewReader(readyMetadata+test.expectedReady),
|
||||
"certmanager_certificate_ready_status",
|
||||
); err != nil {
|
||||
t.Errorf("unexpected collecting result:\n%s", err)
|
||||
@ -151,118 +145,112 @@ func TestUpdateCertificateReadyStatus(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanUp(t *testing.T) {
|
||||
const metadataExpiry = `
|
||||
# HELP certmanager_certificate_expiration_timestamp_seconds The date after which the certificate expires. Expressed as a Unix Epoch Time.
|
||||
# TYPE certmanager_certificate_expiration_timestamp_seconds gauge
|
||||
`
|
||||
func TestCertificateCache(t *testing.T) {
|
||||
m := New(logtesting.TestLogger{T: t})
|
||||
|
||||
const metadataReady = `
|
||||
# HELP certmanager_certificate_ready_status The ready status of the certificate.
|
||||
# TYPE certmanager_certificate_ready_status gauge
|
||||
`
|
||||
type testT struct {
|
||||
active map[*v1alpha2.Certificate]*x509.Certificate
|
||||
inactive map[*v1alpha2.Certificate]*x509.Certificate
|
||||
expectedExpiry string
|
||||
expectedReady string
|
||||
crt1 := gen.Certificate("crt1",
|
||||
gen.SetCertificateUID("uid-1"),
|
||||
gen.SetCertificateNotAfter(metav1.Time{
|
||||
Time: time.Unix(100, 0),
|
||||
}),
|
||||
gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
|
||||
Type: cmapi.CertificateConditionReady,
|
||||
Status: cmmeta.ConditionUnknown,
|
||||
}),
|
||||
)
|
||||
crt2 := gen.Certificate("crt2",
|
||||
gen.SetCertificateUID("uid-2"),
|
||||
gen.SetCertificateNotAfter(metav1.Time{
|
||||
Time: time.Unix(200, 0),
|
||||
}),
|
||||
gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
|
||||
Type: cmapi.CertificateConditionReady,
|
||||
Status: cmmeta.ConditionTrue,
|
||||
}),
|
||||
)
|
||||
crt3 := gen.Certificate("crt3",
|
||||
gen.SetCertificateUID("uid-3"),
|
||||
gen.SetCertificateNotAfter(metav1.Time{
|
||||
Time: time.Unix(300, 0),
|
||||
}),
|
||||
gen.SetCertificateStatusCondition(cmapi.CertificateCondition{
|
||||
Type: cmapi.CertificateConditionReady,
|
||||
Status: cmmeta.ConditionFalse,
|
||||
}),
|
||||
)
|
||||
|
||||
// Observe all three Certificate metrics
|
||||
m.UpdateCertificate(context.TODO(), crt1)
|
||||
m.UpdateCertificate(context.TODO(), crt2)
|
||||
m.UpdateCertificate(context.TODO(), crt3)
|
||||
|
||||
// Check all three metrics exist
|
||||
if err := testutil.CollectAndCompare(m.certificateReadyStatus,
|
||||
strings.NewReader(readyMetadata+`
|
||||
certmanager_certificate_ready_status{condition="False",name="crt1",namespace="default-unit-test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="False",name="crt2",namespace="default-unit-test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="False",name="crt3",namespace="default-unit-test-ns"} 1
|
||||
certmanager_certificate_ready_status{condition="True",name="crt1",namespace="default-unit-test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="True",name="crt2",namespace="default-unit-test-ns"} 1
|
||||
certmanager_certificate_ready_status{condition="True",name="crt3",namespace="default-unit-test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="crt1",namespace="default-unit-test-ns"} 1
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="crt2",namespace="default-unit-test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="crt3",namespace="default-unit-test-ns"} 0
|
||||
`),
|
||||
"certmanager_certificate_ready_status",
|
||||
); err != nil {
|
||||
t.Errorf("unexpected collecting result:\n%s", err)
|
||||
}
|
||||
tests := map[string]testT{
|
||||
"inactive certificate metrics cleaned up while active certificate metrics kept": {
|
||||
active: map[*v1alpha2.Certificate]*x509.Certificate{
|
||||
buildCertificate("active", "default", cmmeta.ConditionTrue): {
|
||||
// fixed expiry time for testing
|
||||
NotAfter: time.Unix(2208988804, 0),
|
||||
},
|
||||
},
|
||||
inactive: map[*v1alpha2.Certificate]*x509.Certificate{
|
||||
buildCertificate("inactive", "default", cmmeta.ConditionTrue): {
|
||||
// fixed expiry time for testing
|
||||
NotAfter: time.Unix(2208988804, 0),
|
||||
},
|
||||
},
|
||||
expectedExpiry: `
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="active",namespace="default"} 2.208988804e+09
|
||||
`,
|
||||
expectedReady: `
|
||||
certmanager_certificate_ready_status{condition="False",name="active",namespace="default"} 0
|
||||
certmanager_certificate_ready_status{condition="True",name="active",namespace="default"} 1
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="active",namespace="default"} 0
|
||||
`,
|
||||
},
|
||||
"no metrics cleaned up when only active certificate metrics": {
|
||||
active: map[*v1alpha2.Certificate]*x509.Certificate{
|
||||
buildCertificate("active", "default", cmmeta.ConditionTrue): {
|
||||
// fixed expiry time for testing
|
||||
NotAfter: time.Unix(2208988804, 0),
|
||||
},
|
||||
buildCertificate("also-active", "default", cmmeta.ConditionTrue): {
|
||||
// fixed expiry time for testing
|
||||
NotAfter: time.Unix(2208988804, 0),
|
||||
},
|
||||
},
|
||||
inactive: map[*v1alpha2.Certificate]*x509.Certificate{},
|
||||
expectedExpiry: `
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="active",namespace="default"} 2.208988804e+09
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="also-active",namespace="default"} 2.208988804e+09
|
||||
`,
|
||||
expectedReady: `
|
||||
certmanager_certificate_ready_status{condition="False",name="active",namespace="default"} 0
|
||||
certmanager_certificate_ready_status{condition="False",name="also-active",namespace="default"} 0
|
||||
certmanager_certificate_ready_status{condition="True",name="active",namespace="default"} 1
|
||||
certmanager_certificate_ready_status{condition="True",name="also-active",namespace="default"} 1
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="active",namespace="default"} 0
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="also-active",namespace="default"} 0
|
||||
`,
|
||||
},
|
||||
"all metrics cleaned up when only inactive certificate metrics": {
|
||||
active: map[*v1alpha2.Certificate]*x509.Certificate{},
|
||||
inactive: map[*v1alpha2.Certificate]*x509.Certificate{
|
||||
buildCertificate("inactive", "default", cmmeta.ConditionTrue): {
|
||||
// fixed expiry time for testing
|
||||
NotAfter: time.Unix(2208988804, 0),
|
||||
},
|
||||
buildCertificate("also-inactive", "default", cmmeta.ConditionTrue): {
|
||||
// fixed expiry time for testing
|
||||
NotAfter: time.Unix(2208988804, 0),
|
||||
},
|
||||
},
|
||||
expectedExpiry: "",
|
||||
expectedReady: "",
|
||||
},
|
||||
if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds,
|
||||
strings.NewReader(expiryMetadata+`
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="crt1",namespace="default-unit-test-ns"} 100
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="crt2",namespace="default-unit-test-ns"} 200
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="crt3",namespace="default-unit-test-ns"} 300
|
||||
`),
|
||||
"certmanager_certificate_expiration_timestamp_seconds",
|
||||
); err != nil {
|
||||
t.Errorf("unexpected collecting result:\n%s", err)
|
||||
}
|
||||
for n, test := range tests {
|
||||
t.Run(n, func(t *testing.T) {
|
||||
defer cleanUpCertificates(nil)
|
||||
|
||||
var activeCrts []*v1alpha2.Certificate
|
||||
for crt, cert := range test.active {
|
||||
updateX509Expiry(crt, cert)
|
||||
updateCertificateReadyStatus(crt, getReadyConditionStatus(crt))
|
||||
activeCrts = append(activeCrts, crt)
|
||||
}
|
||||
for crt, cert := range test.inactive {
|
||||
updateCertificateReadyStatus(crt, getReadyConditionStatus(crt))
|
||||
updateX509Expiry(crt, cert)
|
||||
}
|
||||
// Remove second certificate and check not exists
|
||||
m.RemoveCertificate("default-unit-test-ns/crt2")
|
||||
if err := testutil.CollectAndCompare(m.certificateReadyStatus,
|
||||
strings.NewReader(readyMetadata+`
|
||||
certmanager_certificate_ready_status{condition="False",name="crt1",namespace="default-unit-test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="False",name="crt3",namespace="default-unit-test-ns"} 1
|
||||
certmanager_certificate_ready_status{condition="True",name="crt1",namespace="default-unit-test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="True",name="crt3",namespace="default-unit-test-ns"} 0
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="crt1",namespace="default-unit-test-ns"} 1
|
||||
certmanager_certificate_ready_status{condition="Unknown",name="crt3",namespace="default-unit-test-ns"} 0
|
||||
`),
|
||||
"certmanager_certificate_ready_status",
|
||||
); err != nil {
|
||||
t.Errorf("unexpected collecting result:\n%s", err)
|
||||
}
|
||||
if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds,
|
||||
strings.NewReader(expiryMetadata+`
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="crt1",namespace="default-unit-test-ns"} 100
|
||||
certmanager_certificate_expiration_timestamp_seconds{name="crt3",namespace="default-unit-test-ns"} 300
|
||||
`),
|
||||
"certmanager_certificate_expiration_timestamp_seconds",
|
||||
); err != nil {
|
||||
t.Errorf("unexpected collecting result:\n%s", err)
|
||||
}
|
||||
|
||||
cleanUpCertificates(activeCrts)
|
||||
|
||||
if err := testutil.CollectAndCompare(
|
||||
CertificateExpiryTimeSeconds,
|
||||
strings.NewReader(metadataExpiry+test.expectedExpiry),
|
||||
"certmanager_certificate_expiration_timestamp_seconds",
|
||||
); err != nil {
|
||||
t.Errorf("unexpected collecting result:\n%s", err)
|
||||
}
|
||||
|
||||
if err := testutil.CollectAndCompare(
|
||||
CertificateReadyStatus,
|
||||
strings.NewReader(metadataReady+test.expectedReady),
|
||||
"certmanager_certificate_ready_status",
|
||||
); err != nil {
|
||||
t.Errorf("unexpected collecting result:\n%s", err)
|
||||
}
|
||||
})
|
||||
// Remove all Certificates (even is already removed) and observe no Certificates
|
||||
m.RemoveCertificate("default-unit-test-ns/crt1")
|
||||
m.RemoveCertificate("default-unit-test-ns/crt2")
|
||||
m.RemoveCertificate("default-unit-test-ns/crt3")
|
||||
if err := testutil.CollectAndCompare(m.certificateReadyStatus,
|
||||
strings.NewReader(readyMetadata),
|
||||
"certmanager_certificate_ready_status",
|
||||
); err != nil {
|
||||
t.Errorf("unexpected collecting result:\n%s", err)
|
||||
}
|
||||
if err := testutil.CollectAndCompare(m.certificateExpiryTimeSeconds,
|
||||
strings.NewReader(expiryMetadata),
|
||||
"certmanager_certificate_expiration_timestamp_seconds",
|
||||
); err != nil {
|
||||
t.Errorf("unexpected collecting result:\n%s", err)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user