Add metrics server to the cainjector

Signed-off-by: Richard Wall <richard.wall@venafi.com>

cmd/cainjector/app/controller.go

@@ -18,6 +18,7 @@ package app
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net"
 	"net/http"
@@ -30,7 +31,9 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	kscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	ciphers "k8s.io/component-base/cli/flag"
 	apireg "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -39,9 +42,12 @@ import (
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	config "github.com/cert-manager/cert-manager/internal/apis/config/cainjector"
+	"github.com/cert-manager/cert-manager/internal/apis/config/shared"
 	cmscheme "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned/scheme"
 	"github.com/cert-manager/cert-manager/pkg/controller/cainjector"
 	logf "github.com/cert-manager/cert-manager/pkg/logs"
+	cmservertls "github.com/cert-manager/cert-manager/pkg/server/tls"
+	"github.com/cert-manager/cert-manager/pkg/server/tls/authority"
 	"github.com/cert-manager/cert-manager/pkg/util"
 	"github.com/cert-manager/cert-manager/pkg/util/profiling"
 )
@@ -60,6 +66,8 @@ const (
 func Run(opts *config.CAInjectorConfiguration, ctx context.Context) error {
 	log := logf.FromContext(ctx)
 
+	restConfig := util.RestConfigWithUserAgent(ctrl.GetConfigOrDie(), "cainjector")
+
 	var defaultNamespaces map[string]cache.Config
 	if opts.Namespace != "" {
 		// If a namespace has been provided, only watch resources in that namespace
@@ -68,6 +76,12 @@ func Run(opts *config.CAInjectorConfiguration, ctx context.Context) error {
 		}
 	}
 
+	metricsServerCertificateSource := buildCertificateSource(opts.MetricsTLSConfig, restConfig)
+	metricsServerOptions, err := buildMetricsServerOptions(opts, metricsServerCertificateSource)
+	if err != nil {
+		return err
+	}
+
 	scheme := runtime.NewScheme()
 	kscheme.AddToScheme(scheme)
 	cmscheme.AddToScheme(scheme)
@@ -75,7 +89,7 @@ func Run(opts *config.CAInjectorConfiguration, ctx context.Context) error {
 	apireg.AddToScheme(scheme)
 
 	mgr, err := ctrl.NewManager(
-		util.RestConfigWithUserAgent(ctrl.GetConfigOrDie(), "cainjector"),
+		restConfig,
 		ctrl.Options{
 			Scheme: scheme,
 			Cache: cache.Options{
@@ -130,12 +144,18 @@ func Run(opts *config.CAInjectorConfiguration, ctx context.Context) error {
 			LeaseDuration:                 &opts.LeaderElectionConfig.LeaseDuration,
 			RenewDeadline:                 &opts.LeaderElectionConfig.RenewDeadline,
 			RetryPeriod:                   &opts.LeaderElectionConfig.RetryPeriod,
-			Metrics:                       metricsserver.Options{BindAddress: "0"},
+			Metrics:                       *metricsServerOptions,
 		})
 	if err != nil {
 		return fmt.Errorf("error creating manager: %v", err)
 	}
 
+	if metricsServerCertificateSource != nil {
+		if err := mgr.Add(metricsServerCertificateSource); err != nil {
+			return err
+		}
+	}
+
 	// if a PprofAddr is provided, start the pprof listener
 	if opts.EnablePprof {
 		pprofListener, err := net.Listen("tcp", opts.PprofAddress)
@@ -241,3 +261,50 @@ func (runnableNoLeaderElectionFunc) NeedLeaderElection() bool {
 var _ manager.Runnable = runnableNoLeaderElectionFunc(nil)
 
 var _ manager.LeaderElectionRunnable = runnableNoLeaderElectionFunc(nil)
+
+func buildMetricsServerOptions(opts *config.CAInjectorConfiguration, cs cmservertls.CertificateSource) (*metricsserver.Options, error) {
+	msOptions := metricsserver.Options{
+		BindAddress: opts.MetricsListenAddress,
+	}
+	if cs != nil {
+		metricsCipherSuites, err := ciphers.TLSCipherSuites(opts.MetricsTLSConfig.CipherSuites)
+		if err != nil {
+			return nil, err
+		}
+		metricsMinVersion, err := ciphers.TLSVersion(opts.MetricsTLSConfig.MinTLSVersion)
+		if err != nil {
+			return nil, err
+		}
+		msOptions.SecureServing = true
+		msOptions.TLSOpts = []func(*tls.Config){
+			func(cfg *tls.Config) {
+				cfg.CipherSuites = metricsCipherSuites
+				cfg.MinVersion = metricsMinVersion
+				cfg.GetCertificate = cs.GetCertificate
+			},
+		}
+	}
+	return &msOptions, nil
+}
+
+func buildCertificateSource(tlsConfig shared.TLSConfig, restCfg *rest.Config) cmservertls.CertificateSource {
+	switch {
+	case tlsConfig.FilesystemConfigProvided():
+		return &cmservertls.FileCertificateSource{
+			CertPath: tlsConfig.Filesystem.CertFile,
+			KeyPath:  tlsConfig.Filesystem.KeyFile,
+		}
+
+	case tlsConfig.DynamicConfigProvided():
+		return &cmservertls.DynamicSource{
+			DNSNames: tlsConfig.Dynamic.DNSNames,
+			Authority: &authority.DynamicAuthority{
+				SecretNamespace: tlsConfig.Dynamic.SecretNamespace,
+				SecretName:      tlsConfig.Dynamic.SecretName,
+				LeafDuration:    tlsConfig.Dynamic.LeafDuration,
+				RESTConfig:      restCfg,
+			},
+		}
+	}
+	return nil
+}

cmd/cainjector/app/options/options.go

@@ -112,6 +112,25 @@ func AddConfigFlags(fs *pflag.FlagSet, c *config.CAInjectorConfiguration) {
 
 	logf.AddFlags(&c.Logging, fs)
 
+	fs.StringVar(&c.MetricsListenAddress, "metrics-listen-address", c.MetricsListenAddress, "The host and port that the metrics endpoint should listen on. The value '0' disables the metrics server")
+	fs.StringVar(&c.MetricsTLSConfig.Filesystem.CertFile, "metrics-tls-cert-file", c.MetricsTLSConfig.Filesystem.CertFile, "path to the file containing the TLS certificate to serve metrics with")
+	fs.StringVar(&c.MetricsTLSConfig.Filesystem.KeyFile, "metrics-tls-private-key-file", c.MetricsTLSConfig.Filesystem.KeyFile, "path to the file containing the TLS private key to serve metrics with")
+
+	fs.DurationVar(&c.MetricsTLSConfig.Dynamic.LeafDuration, "metrics-dynamic-serving-leaf-duration", c.MetricsTLSConfig.Dynamic.LeafDuration, "leaf duration of metrics serving certificates")
+	fs.StringVar(&c.MetricsTLSConfig.Dynamic.SecretNamespace, "metrics-dynamic-serving-ca-secret-namespace", c.MetricsTLSConfig.Dynamic.SecretNamespace, "namespace of the secret used to store the CA that signs metrics serving certificates")
+	fs.StringVar(&c.MetricsTLSConfig.Dynamic.SecretName, "metrics-dynamic-serving-ca-secret-name", c.MetricsTLSConfig.Dynamic.SecretName, "name of the secret used to store the CA that signs serving certificates")
+	fs.StringSliceVar(&c.MetricsTLSConfig.Dynamic.DNSNames, "metrics-dynamic-serving-dns-names", c.MetricsTLSConfig.Dynamic.DNSNames, "DNS names that should be present on certificates generated by the metrics dynamic serving CA")
+
+	tlsCipherPossibleValues := cliflag.TLSCipherPossibleValues()
+	fs.StringSliceVar(&c.MetricsTLSConfig.CipherSuites, "metrics-tls-cipher-suites", c.MetricsTLSConfig.CipherSuites,
+		"Comma-separated list of cipher suites for the metrics server. "+
+			"If omitted, the default Go cipher suites will be used.  "+
+			"Possible values: "+strings.Join(tlsCipherPossibleValues, ","))
+	tlsPossibleVersions := cliflag.TLSPossibleVersions()
+	fs.StringVar(&c.MetricsTLSConfig.MinTLSVersion, "metrics-tls-min-version", c.MetricsTLSConfig.MinTLSVersion,
+		"Minimum TLS version supported by the metrics server. If omitted, the default Go minimum version will be used. "+
+			"Possible values: "+strings.Join(tlsPossibleVersions, ", "))
+
 	// The controller-runtime flag (--kubeconfig) that we need
 	// relies on the "flag" package but we use "spf13/pflag".
 	var controllerRuntimeFlags flag.FlagSet

cmd/cainjector/go.mod

@@ -15,6 +15,9 @@ replace github.com/prometheus/common => github.com/prometheus/common v0.46.0
 
 replace github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.18.0
 
+// Can be removed once github.com/go-ldap/ldap/v3 releases a version that requires this version.
+replace github.com/go-asn1-ber/asn1-ber => github.com/go-asn1-ber/asn1-ber v1.5.6
+
 replace github.com/cert-manager/cert-manager => ../../
 
 require (

internal/apis/config/cainjector/fuzzer/fuzzer.go

@@ -46,6 +46,9 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} {
 			if s.LeaderElectionConfig.RetryPeriod == 0 {
 				s.LeaderElectionConfig.RetryPeriod = 1234
 			}
+			if s.MetricsListenAddress == "" {
+				s.MetricsListenAddress = "something:1234"
+			}
 
 			logsapi.SetRecommendedLoggingConfiguration(&s.Logging)
 		},

internal/apis/config/cainjector/types.go

@@ -61,6 +61,14 @@ type CAInjectorConfiguration struct {
 	// featureGates is a map of feature names to bools that enable or disable experimental
 	// features.
 	FeatureGates map[string]bool
+
+	// The host and port that the metrics endpoint should listen on.
+	// The value "0" disables the metrics server.
+	// Defaults to '0.0.0.0:9402'.
+	MetricsListenAddress string
+
+	// Metrics endpoint TLS config
+	MetricsTLSConfig shared.TLSConfig
 }
 
 type EnableDataSourceConfig struct {

internal/apis/config/cainjector/v1alpha1/defaults.go

@@ -24,6 +24,8 @@ import (
 	"github.com/cert-manager/cert-manager/pkg/apis/config/cainjector/v1alpha1"
 )
 
+const defaultPrometheusMetricsServerAddress = "0.0.0.0:9402"
+
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
 	return RegisterDefaults(scheme)
 }
@@ -33,6 +35,10 @@ func SetDefaults_CAInjectorConfiguration(obj *v1alpha1.CAInjectorConfiguration)
 		obj.PprofAddress = "localhost:6060"
 	}
 
+	if obj.MetricsListenAddress == "" {
+		obj.MetricsListenAddress = defaultPrometheusMetricsServerAddress
+	}
+
 	logsapi.SetRecommendedLoggingConfiguration(&obj.Logging)
 }
 

internal/apis/config/cainjector/v1alpha1/testdata/defaults.json

@@ -29,5 +29,12 @@
 				"infoBufferSize": "0"
 			}
 		}
+	},
+	"metricsListenAddress": "0.0.0.0:9402",
+	"metricsTLSConfig": {
+		"filesystem": {},
+		"dynamic": {
+			"leafDuration": "168h0m0s"
+		}
 	}
 }

pkg/apis/config/cainjector/v1alpha1/types.go

@@ -64,6 +64,14 @@ type CAInjectorConfiguration struct {
 	// features.
 	// +optional
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
+
+	// The host and port that the metrics endpoint should listen on.
+	// The value "0" disables the metrics server.
+	// Defaults to '0.0.0.0:9402'.
+	MetricsListenAddress string `json:"metricsListenAddress,omitempty"`
+
+	// metricsTLSConfig is used to configure the metrics server TLS settings.
+	MetricsTLSConfig sharedv1alpha1.TLSConfig `json:"metricsTLSConfig"`
 }
 
 type EnableDataSourceConfig struct {

pkg/cainjector/configfile/configfile.go

@@ -79,6 +79,8 @@ func (cfg *CAInjectorConfigFile) GetPathRefs() ([]*string, error) {
 // passing the configuration to the application. This method must be kept up to date as new fields are added.
 func CAInjectorConfigurationPathRefs(cfg *config.CAInjectorConfiguration) ([]*string, error) {
 	return []*string{
+		&cfg.MetricsTLSConfig.Filesystem.KeyFile,
+		&cfg.MetricsTLSConfig.Filesystem.CertFile,
 		&cfg.KubeConfig,
 	}, nil
 }