diff --git a/deploy/charts/cert-manager/README.template.md b/deploy/charts/cert-manager/README.template.md index 3cfd64b45..d7f33eab5 100644 --- a/deploy/charts/cert-manager/README.template.md +++ b/deploy/charts/cert-manager/README.template.md @@ -373,14 +373,14 @@ config: StableCertificateRequestName: true UseCertificateRequestBasicConstraints: true ValidateCAA: true + # Configure the metrics server for TLS + # See https://cert-manager.io/docs/devops-tips/prometheus-metrics/#tls metricsTLSConfig: dynamic: secretNamespace: "cert-manager" secretName: "cert-manager-metrics-ca" dnsNames: - cert-manager-metrics - - cert-manager-metrics.cert-manager - - cert-manager-metrics.cert-manager.svc ``` #### **dns01RecursiveNameservers** ~ `string` > Default value: @@ -660,9 +660,9 @@ enableServiceLinks indicates whether information about services should be inject > true > ``` -Enable Prometheus monitoring for the cert-manager controller to use with the. Prometheus Operator. If this option is enabled without enabling `prometheus.servicemonitor.enabled` or -`prometheus.podmonitor.enabled`, 'prometheus.io' annotations are added to the cert-manager Deployment -resources. Additionally, a service is created which can be used together with your own ServiceMonitor (managed outside of this Helm chart). Otherwise, a ServiceMonitor/ PodMonitor is created. +Enable Prometheus monitoring for the cert-manager controller and webhook. If you use the Prometheus Operator, set prometheus.podmonitor.enabled or prometheus.servicemonitor.enabled, to create a PodMonitor or a +ServiceMonitor resource. +Otherwise, 'prometheus.io' annotations are added to the cert-manager and cert-manager-webhook Deployments. Note that you can not enable both PodMonitor and ServiceMonitor as they are mutually exclusive. Enabling both will result in a error. #### **prometheus.servicemonitor.enabled** ~ `bool` > Default value: > ```yaml @@ -828,6 +828,15 @@ endpointAdditionalProperties: sourceLabels: - __meta_kubernetes_pod_node_name targetLabel: instance + # Configure the PodMonitor for TLS connections + # See https://cert-manager.io/docs/devops-tips/prometheus-metrics/#tls + scheme: https + tlsConfig: + serverName: cert-manager-metrics + ca: + secret: + name: cert-manager-metrics-ca + key: "tls.crt" ``` @@ -878,6 +887,14 @@ kind: WebhookConfiguration # This should be uncommented and set as a default by the chart once # the apiVersion of WebhookConfiguration graduates beyond v1alpha1. securePort: 10250 +# Configure the metrics server for TLS +# See https://cert-manager.io/docs/devops-tips/prometheus-metrics/#tls +metricsTLSConfig: + dynamic: + secretNamespace: "cert-manager" + secretName: "cert-manager-metrics-ca" + dnsNames: + - cert-manager-metrics ``` #### **webhook.strategy** ~ `object` > Default value: diff --git a/deploy/charts/cert-manager/templates/podmonitor.yaml b/deploy/charts/cert-manager/templates/podmonitor.yaml index 175460ebe..b1313df37 100644 --- a/deploy/charts/cert-manager/templates/podmonitor.yaml +++ b/deploy/charts/cert-manager/templates/podmonitor.yaml @@ -29,10 +29,21 @@ metadata: spec: jobLabel: {{ template "cert-manager.fullname" . }} selector: - matchLabels: - app.kubernetes.io/name: {{ template "cert-manager.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: "controller" + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - {{ template "cert-manager.name" . }} + - {{ include "webhook.name" . }} + - key: app.kubernetes.io/instance + operator: In + values: + - {{ .Release.Name }} + - key: app.kubernetes.io/component + operator: In + values: + - controller + - webhook {{- if .Values.prometheus.podmonitor.namespace }} namespaceSelector: matchNames: diff --git a/deploy/charts/cert-manager/templates/servicemonitor.yaml b/deploy/charts/cert-manager/templates/servicemonitor.yaml index b63886077..679216c39 100644 --- a/deploy/charts/cert-manager/templates/servicemonitor.yaml +++ b/deploy/charts/cert-manager/templates/servicemonitor.yaml @@ -29,10 +29,21 @@ metadata: spec: jobLabel: {{ template "cert-manager.fullname" . }} selector: - matchLabels: - app.kubernetes.io/name: {{ template "cert-manager.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: "controller" + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - {{ template "cert-manager.name" . }} + - {{ include "webhook.name" . }} + - key: app.kubernetes.io/instance + operator: In + values: + - {{ .Release.Name }} + - key: app.kubernetes.io/component + operator: In + values: + - controller + - webhook {{- if .Values.prometheus.servicemonitor.namespace }} namespaceSelector: matchNames: diff --git a/deploy/charts/cert-manager/templates/webhook-deployment.yaml b/deploy/charts/cert-manager/templates/webhook-deployment.yaml index ae5399e90..4b8d24972 100644 --- a/deploy/charts/cert-manager/templates/webhook-deployment.yaml +++ b/deploy/charts/cert-manager/templates/webhook-deployment.yaml @@ -43,6 +43,14 @@ spec: annotations: {{- toYaml . | nindent 8 }} {{- end }} + {{- if and .Values.prometheus.enabled (not (or .Values.prometheus.servicemonitor.enabled .Values.prometheus.podmonitor.enabled)) }} + {{- if not .Values.podAnnotations }} + annotations: + {{- end }} + prometheus.io/path: "/metrics" + prometheus.io/scrape: 'true' + prometheus.io/port: '9402' + {{- end }} spec: serviceAccountName: {{ template "webhook.serviceAccountName" . }} {{- if hasKey .Values.webhook "automountServiceAccountToken" }} @@ -95,6 +103,9 @@ spec: {{- with .Values.webhook.extraArgs }} {{- toYaml . | nindent 10 }} {{- end }} + {{- if not .Values.prometheus.enabled }} + - --metrics-listen-address=0 + {{- end }} ports: - name: https protocol: TCP @@ -112,6 +123,11 @@ spec: {{- else }} containerPort: 6080 {{- end }} + {{- if .Values.prometheus.enabled }} + - containerPort: 9402 + name: http-metrics + protocol: TCP + {{- end }} livenessProbe: httpGet: path: /livez diff --git a/deploy/charts/cert-manager/templates/webhook-rbac.yaml b/deploy/charts/cert-manager/templates/webhook-rbac.yaml index b075ffd46..2b32d709f 100644 --- a/deploy/charts/cert-manager/templates/webhook-rbac.yaml +++ b/deploy/charts/cert-manager/templates/webhook-rbac.yaml @@ -15,6 +15,15 @@ rules: resources: ["secrets"] resourceNames: - '{{ template "webhook.fullname" . }}-ca' + {{- $certmanagerNamespace := include "cert-manager.namespace" . }} + {{- with (.Values.webhook.config.metricsTLSConfig).dynamic }} + {{- if $certmanagerNamespace | eq .secretNamespace }} + # Allow webhook to read and update the metrics CA Secret when dynamic TLS is + # enabled for the metrics server and if the Secret is configured to be in the + # same namespace as cert-manager. + - {{ .secretName | quote }} + {{- end }} + {{- end }} verbs: ["get", "list", "watch", "update"] # It's not possible to grant CREATE permission on a single resourceName. - apiGroups: [""] diff --git a/deploy/charts/cert-manager/templates/webhook-service.yaml b/deploy/charts/cert-manager/templates/webhook-service.yaml index 86d47f164..cd5010f20 100644 --- a/deploy/charts/cert-manager/templates/webhook-service.yaml +++ b/deploy/charts/cert-manager/templates/webhook-service.yaml @@ -32,6 +32,12 @@ spec: port: 443 protocol: TCP targetPort: "https" +{{- if and .Values.prometheus.enabled (not .Values.prometheus.podmonitor.enabled) }} + - name: metrics + port: 9402 + protocol: TCP + targetPort: "http-metrics" +{{- end }} selector: app.kubernetes.io/name: {{ include "webhook.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/deploy/charts/cert-manager/values.schema.json b/deploy/charts/cert-manager/values.schema.json index 9eb0499a7..beae5ad52 100644 --- a/deploy/charts/cert-manager/values.schema.json +++ b/deploy/charts/cert-manager/values.schema.json @@ -554,7 +554,7 @@ }, "helm-values.config": { "default": {}, - "description": "This property is used to configure options for the controller pod. This allows setting options that would usually be provided using flags.\n\nIf `apiVersion` and `kind` are unspecified they default to the current latest version (currently `controller.config.cert-manager.io/v1alpha1`). You can pin the version by specifying the `apiVersion` yourself.\n\nFor example:\nconfig:\n apiVersion: controller.config.cert-manager.io/v1alpha1\n kind: ControllerConfiguration\n logging:\n verbosity: 2\n format: text\n leaderElectionConfig:\n namespace: kube-system\n kubernetesAPIQPS: 9000\n kubernetesAPIBurst: 9000\n numberOfConcurrentWorkers: 200\n featureGates:\n AdditionalCertificateOutputFormats: true\n DisallowInsecureCSRUsageDefinition: true\n ExperimentalCertificateSigningRequestControllers: true\n ExperimentalGatewayAPISupport: true\n LiteralCertificateSubject: true\n SecretsFilteredCaching: true\n ServerSideApply: true\n StableCertificateRequestName: true\n UseCertificateRequestBasicConstraints: true\n ValidateCAA: true\n metricsTLSConfig:\n dynamic:\n secretNamespace: \"cert-manager\"\n secretName: \"cert-manager-metrics-ca\"\n dnsNames:\n - cert-manager-metrics\n - cert-manager-metrics.cert-manager\n - cert-manager-metrics.cert-manager.svc", + "description": "This property is used to configure options for the controller pod. This allows setting options that would usually be provided using flags.\n\nIf `apiVersion` and `kind` are unspecified they default to the current latest version (currently `controller.config.cert-manager.io/v1alpha1`). You can pin the version by specifying the `apiVersion` yourself.\n\nFor example:\nconfig:\n apiVersion: controller.config.cert-manager.io/v1alpha1\n kind: ControllerConfiguration\n logging:\n verbosity: 2\n format: text\n leaderElectionConfig:\n namespace: kube-system\n kubernetesAPIQPS: 9000\n kubernetesAPIBurst: 9000\n numberOfConcurrentWorkers: 200\n featureGates:\n AdditionalCertificateOutputFormats: true\n DisallowInsecureCSRUsageDefinition: true\n ExperimentalCertificateSigningRequestControllers: true\n ExperimentalGatewayAPISupport: true\n LiteralCertificateSubject: true\n SecretsFilteredCaching: true\n ServerSideApply: true\n StableCertificateRequestName: true\n UseCertificateRequestBasicConstraints: true\n ValidateCAA: true\n # Configure the metrics server for TLS\n # See https://cert-manager.io/docs/devops-tips/prometheus-metrics/#tls\n metricsTLSConfig:\n dynamic:\n secretNamespace: \"cert-manager\"\n secretName: \"cert-manager-metrics-ca\"\n dnsNames:\n - cert-manager-metrics", "type": "object" }, "helm-values.containerSecurityContext": { @@ -976,7 +976,7 @@ }, "helm-values.prometheus.enabled": { "default": true, - "description": "Enable Prometheus monitoring for the cert-manager controller to use with the. Prometheus Operator. If this option is enabled without enabling `prometheus.servicemonitor.enabled` or\n`prometheus.podmonitor.enabled`, 'prometheus.io' annotations are added to the cert-manager Deployment\nresources. Additionally, a service is created which can be used together with your own ServiceMonitor (managed outside of this Helm chart). Otherwise, a ServiceMonitor/ PodMonitor is created.", + "description": "Enable Prometheus monitoring for the cert-manager controller and webhook. If you use the Prometheus Operator, set prometheus.podmonitor.enabled or prometheus.servicemonitor.enabled, to create a PodMonitor or a\nServiceMonitor resource.\nOtherwise, 'prometheus.io' annotations are added to the cert-manager and cert-manager-webhook Deployments. Note that you can not enable both PodMonitor and ServiceMonitor as they are mutually exclusive. Enabling both will result in a error.", "type": "boolean" }, "helm-values.prometheus.podmonitor": { @@ -1027,7 +1027,7 @@ }, "helm-values.prometheus.podmonitor.endpointAdditionalProperties": { "default": {}, - "description": "EndpointAdditionalProperties allows setting additional properties on the endpoint such as relabelings, metricRelabelings etc.\n\nFor example:\nendpointAdditionalProperties:\n relabelings:\n - action: replace\n sourceLabels:\n - __meta_kubernetes_pod_node_name\n targetLabel: instance", + "description": "EndpointAdditionalProperties allows setting additional properties on the endpoint such as relabelings, metricRelabelings etc.\n\nFor example:\nendpointAdditionalProperties:\n relabelings:\n - action: replace\n sourceLabels:\n - __meta_kubernetes_pod_node_name\n targetLabel: instance\n # Configure the PodMonitor for TLS connections\n # See https://cert-manager.io/docs/devops-tips/prometheus-metrics/#tls\n scheme: https\n tlsConfig:\n serverName: cert-manager-metrics\n ca:\n secret:\n name: cert-manager-metrics-ca\n key: \"tls.crt\"", "type": "object" }, "helm-values.prometheus.podmonitor.honorLabels": { @@ -1683,7 +1683,7 @@ }, "helm-values.webhook.config": { "default": {}, - "description": "This is used to configure options for the webhook pod. This allows setting options that would usually be provided using flags.\n\nIf `apiVersion` and `kind` are unspecified they default to the current latest version (currently `webhook.config.cert-manager.io/v1alpha1`). You can pin the version by specifying the `apiVersion` yourself.\n\nFor example:\napiVersion: webhook.config.cert-manager.io/v1alpha1\nkind: WebhookConfiguration\n# The port that the webhook listens on for requests.\n# In GKE private clusters, by default Kubernetes apiservers are allowed to\n# talk to the cluster nodes only on 443 and 10250. Configuring\n# securePort: 10250 therefore will work out-of-the-box without needing to add firewall\n# rules or requiring NET_BIND_SERVICE capabilities to bind port numbers < 1000.\n# This should be uncommented and set as a default by the chart once\n# the apiVersion of WebhookConfiguration graduates beyond v1alpha1.\nsecurePort: 10250", + "description": "This is used to configure options for the webhook pod. This allows setting options that would usually be provided using flags.\n\nIf `apiVersion` and `kind` are unspecified they default to the current latest version (currently `webhook.config.cert-manager.io/v1alpha1`). You can pin the version by specifying the `apiVersion` yourself.\n\nFor example:\napiVersion: webhook.config.cert-manager.io/v1alpha1\nkind: WebhookConfiguration\n# The port that the webhook listens on for requests.\n# In GKE private clusters, by default Kubernetes apiservers are allowed to\n# talk to the cluster nodes only on 443 and 10250. Configuring\n# securePort: 10250 therefore will work out-of-the-box without needing to add firewall\n# rules or requiring NET_BIND_SERVICE capabilities to bind port numbers < 1000.\n# This should be uncommented and set as a default by the chart once\n# the apiVersion of WebhookConfiguration graduates beyond v1alpha1.\nsecurePort: 10250\n# Configure the metrics server for TLS\n# See https://cert-manager.io/docs/devops-tips/prometheus-metrics/#tls\nmetricsTLSConfig:\n dynamic:\n secretNamespace: \"cert-manager\"\n secretName: \"cert-manager-metrics-ca\"\n dnsNames:\n - cert-manager-metrics", "type": "object" }, "helm-values.webhook.containerSecurityContext": { diff --git a/deploy/charts/cert-manager/values.yaml b/deploy/charts/cert-manager/values.yaml index 7b23b4449..a8035c310 100644 --- a/deploy/charts/cert-manager/values.yaml +++ b/deploy/charts/cert-manager/values.yaml @@ -209,8 +209,8 @@ enableCertificateOwnerRef: false # This property is used to configure options for the controller pod. # This allows setting options that would usually be provided using flags. # -# If `apiVersion` and `kind` are unspecified they default to the current latest -# version (currently `controller.config.cert-manager.io/v1alpha1`). You can pin +# If `apiVersion` and `kind` are unspecified they default to the current latest +# version (currently `controller.config.cert-manager.io/v1alpha1`). You can pin # the version by specifying the `apiVersion` yourself. # # For example: @@ -236,14 +236,14 @@ enableCertificateOwnerRef: false # StableCertificateRequestName: true # UseCertificateRequestBasicConstraints: true # ValidateCAA: true +# # Configure the metrics server for TLS +# # See https://cert-manager.io/docs/devops-tips/prometheus-metrics/#tls # metricsTLSConfig: # dynamic: # secretNamespace: "cert-manager" # secretName: "cert-manager-metrics-ca" # dnsNames: # - cert-manager-metrics -# - cert-manager-metrics.cert-manager -# - cert-manager-metrics.cert-manager.svc config: {} # Setting Nameservers for DNS01 Self Check. @@ -482,12 +482,14 @@ enableServiceLinks: false # +docs:section=Prometheus prometheus: - # Enable Prometheus monitoring for the cert-manager controller to use with the - # Prometheus Operator. If this option is enabled without enabling `prometheus.servicemonitor.enabled` or - # `prometheus.podmonitor.enabled`, 'prometheus.io' annotations are added to the cert-manager Deployment - # resources. Additionally, a service is created which can be used together - # with your own ServiceMonitor (managed outside of this Helm chart). - # Otherwise, a ServiceMonitor/ PodMonitor is created. + # Enable Prometheus monitoring for the cert-manager controller and webhook. + # If you use the Prometheus Operator, set prometheus.podmonitor.enabled or + # prometheus.servicemonitor.enabled, to create a PodMonitor or a + # ServiceMonitor resource. + # Otherwise, 'prometheus.io' annotations are added to the cert-manager and + # cert-manager-webhook Deployments. + # Note that you can not enable both PodMonitor and ServiceMonitor as they are + # mutually exclusive. Enabling both will result in a error. enabled: true servicemonitor: @@ -583,6 +585,15 @@ prometheus: # sourceLabels: # - __meta_kubernetes_pod_node_name # targetLabel: instance + # # Configure the PodMonitor for TLS connections + # # See https://cert-manager.io/docs/devops-tips/prometheus-metrics/#tls + # scheme: https + # tlsConfig: + # serverName: cert-manager-metrics + # ca: + # secret: + # name: cert-manager-metrics-ca + # key: "tls.crt" # # +docs:property endpointAdditionalProperties: {} @@ -617,8 +628,8 @@ webhook: # This is used to configure options for the webhook pod. # This allows setting options that would usually be provided using flags. # - # If `apiVersion` and `kind` are unspecified they default to the current latest - # version (currently `webhook.config.cert-manager.io/v1alpha1`). You can pin + # If `apiVersion` and `kind` are unspecified they default to the current latest + # version (currently `webhook.config.cert-manager.io/v1alpha1`). You can pin # the version by specifying the `apiVersion` yourself. # # For example: @@ -632,6 +643,14 @@ webhook: # # This should be uncommented and set as a default by the chart once # # the apiVersion of WebhookConfiguration graduates beyond v1alpha1. # securePort: 10250 + # # Configure the metrics server for TLS + # # See https://cert-manager.io/docs/devops-tips/prometheus-metrics/#tls + # metricsTLSConfig: + # dynamic: + # secretNamespace: "cert-manager" + # secretName: "cert-manager-metrics-ca" + # dnsNames: + # - cert-manager-metrics config: {} # The update strategy for the cert-manager webhook deployment. @@ -970,8 +989,8 @@ cainjector: # This is used to configure options for the cainjector pod. # It allows setting options that are usually provided via flags. # - # If `apiVersion` and `kind` are unspecified they default to the current latest - # version (currently `cainjector.config.cert-manager.io/v1alpha1`). You can pin + # If `apiVersion` and `kind` are unspecified they default to the current latest + # version (currently `cainjector.config.cert-manager.io/v1alpha1`). You can pin # the version by specifying the `apiVersion` yourself. # # For example: diff --git a/internal/apis/config/webhook/fuzzer/fuzzer.go b/internal/apis/config/webhook/fuzzer/fuzzer.go index b20a4fbd4..78c898e20 100644 --- a/internal/apis/config/webhook/fuzzer/fuzzer.go +++ b/internal/apis/config/webhook/fuzzer/fuzzer.go @@ -33,6 +33,9 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { if s.PprofAddress == "" { s.PprofAddress = "something:1234" } + if s.MetricsListenAddress == "" { + s.MetricsListenAddress = "something:1234" + } logsapi.SetRecommendedLoggingConfiguration(&s.Logging) }, diff --git a/internal/apis/config/webhook/types.go b/internal/apis/config/webhook/types.go index ebb20730f..50c72bd50 100644 --- a/internal/apis/config/webhook/types.go +++ b/internal/apis/config/webhook/types.go @@ -62,4 +62,12 @@ type WebhookConfiguration struct { // featureGates is a map of feature names to bools that enable or disable experimental // features. FeatureGates map[string]bool + + // The host and port that the metrics endpoint should listen on. + // The value "0" disables the metrics server. + // Defaults to '0.0.0.0:9402'. + MetricsListenAddress string + + // Metrics endpoint TLS config + MetricsTLSConfig shared.TLSConfig } diff --git a/internal/apis/config/webhook/v1alpha1/defaults.go b/internal/apis/config/webhook/v1alpha1/defaults.go index 700b8aead..10f87ca76 100644 --- a/internal/apis/config/webhook/v1alpha1/defaults.go +++ b/internal/apis/config/webhook/v1alpha1/defaults.go @@ -24,6 +24,8 @@ import ( "github.com/cert-manager/cert-manager/pkg/apis/config/webhook/v1alpha1" ) +const defaultPrometheusMetricsServerAddress = "0.0.0.0:9402" + func addDefaultingFuncs(scheme *runtime.Scheme) error { return RegisterDefaults(scheme) } @@ -39,5 +41,9 @@ func SetDefaults_WebhookConfiguration(obj *v1alpha1.WebhookConfiguration) { obj.PprofAddress = "localhost:6060" } + if obj.MetricsListenAddress == "" { + obj.MetricsListenAddress = defaultPrometheusMetricsServerAddress + } + logsapi.SetRecommendedLoggingConfiguration(&obj.Logging) } diff --git a/internal/apis/config/webhook/v1alpha1/testdata/defaults.json b/internal/apis/config/webhook/v1alpha1/testdata/defaults.json index 3328d3264..decd20943 100644 --- a/internal/apis/config/webhook/v1alpha1/testdata/defaults.json +++ b/internal/apis/config/webhook/v1alpha1/testdata/defaults.json @@ -21,5 +21,12 @@ "infoBufferSize": "0" } } + }, + "metricsListenAddress": "0.0.0.0:9402", + "metricsTLSConfig": { + "filesystem": {}, + "dynamic": { + "leafDuration": "168h0m0s" + } } } \ No newline at end of file diff --git a/internal/apis/config/webhook/v1alpha1/zz_generated.conversion.go b/internal/apis/config/webhook/v1alpha1/zz_generated.conversion.go index d23776b65..4a5a58d88 100644 --- a/internal/apis/config/webhook/v1alpha1/zz_generated.conversion.go +++ b/internal/apis/config/webhook/v1alpha1/zz_generated.conversion.go @@ -68,6 +68,10 @@ func autoConvert_v1alpha1_WebhookConfiguration_To_webhook_WebhookConfiguration(i out.PprofAddress = in.PprofAddress out.Logging = in.Logging out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + out.MetricsListenAddress = in.MetricsListenAddress + if err := sharedv1alpha1.Convert_v1alpha1_TLSConfig_To_shared_TLSConfig(&in.MetricsTLSConfig, &out.MetricsTLSConfig, s); err != nil { + return err + } return nil } @@ -92,6 +96,10 @@ func autoConvert_webhook_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(i out.PprofAddress = in.PprofAddress out.Logging = in.Logging out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + out.MetricsListenAddress = in.MetricsListenAddress + if err := sharedv1alpha1.Convert_shared_TLSConfig_To_v1alpha1_TLSConfig(&in.MetricsTLSConfig, &out.MetricsTLSConfig, s); err != nil { + return err + } return nil } diff --git a/internal/apis/config/webhook/v1alpha1/zz_generated.defaults.go b/internal/apis/config/webhook/v1alpha1/zz_generated.defaults.go index 616dc64db..b51a5bc52 100644 --- a/internal/apis/config/webhook/v1alpha1/zz_generated.defaults.go +++ b/internal/apis/config/webhook/v1alpha1/zz_generated.defaults.go @@ -38,4 +38,5 @@ func RegisterDefaults(scheme *runtime.Scheme) error { func SetObjectDefaults_WebhookConfiguration(in *v1alpha1.WebhookConfiguration) { SetDefaults_WebhookConfiguration(in) sharedv1alpha1.SetDefaults_DynamicServingConfig(&in.TLSConfig.Dynamic) + sharedv1alpha1.SetDefaults_DynamicServingConfig(&in.MetricsTLSConfig.Dynamic) } diff --git a/internal/apis/config/webhook/zz_generated.deepcopy.go b/internal/apis/config/webhook/zz_generated.deepcopy.go index ed819dc7b..775767f48 100644 --- a/internal/apis/config/webhook/zz_generated.deepcopy.go +++ b/internal/apis/config/webhook/zz_generated.deepcopy.go @@ -38,6 +38,7 @@ func (in *WebhookConfiguration) DeepCopyInto(out *WebhookConfiguration) { (*out)[key] = val } } + in.MetricsTLSConfig.DeepCopyInto(&out.MetricsTLSConfig) return } diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index f1d8abc8c..0bac98ca3 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -71,16 +71,20 @@ func NewCertManagerWebhookServer(log logr.Logger, opts config.WebhookConfigurati metainstall.Install(scheme) s := &server.Server{ - ResourceScheme: scheme, - ListenAddr: opts.SecurePort, - HealthzAddr: &opts.HealthzPort, - EnablePprof: opts.EnablePprof, - PprofAddress: opts.PprofAddress, - CertificateSource: buildCertificateSource(log, opts.TLSConfig, restcfg), - CipherSuites: opts.TLSConfig.CipherSuites, - MinTLSVersion: opts.TLSConfig.MinTLSVersion, - ValidationWebhook: admissionHandler, - MutationWebhook: admissionHandler, + ResourceScheme: scheme, + ListenAddr: opts.SecurePort, + HealthzAddr: &opts.HealthzPort, + EnablePprof: opts.EnablePprof, + PprofAddress: opts.PprofAddress, + CertificateSource: buildCertificateSource(log, opts.TLSConfig, restcfg), + CipherSuites: opts.TLSConfig.CipherSuites, + MinTLSVersion: opts.TLSConfig.MinTLSVersion, + ValidationWebhook: admissionHandler, + MutationWebhook: admissionHandler, + MetricsListenAddress: opts.MetricsListenAddress, + MetricsCertificateSource: buildCertificateSource(log, opts.MetricsTLSConfig, restcfg), + MetricsCipherSuites: opts.MetricsTLSConfig.CipherSuites, + MetricsMinTLSVersion: opts.MetricsTLSConfig.MinTLSVersion, } for _, fn := range optionFunctions { fn(s) diff --git a/pkg/apis/config/webhook/v1alpha1/types.go b/pkg/apis/config/webhook/v1alpha1/types.go index 2b5fbb225..38cecb761 100644 --- a/pkg/apis/config/webhook/v1alpha1/types.go +++ b/pkg/apis/config/webhook/v1alpha1/types.go @@ -64,4 +64,12 @@ type WebhookConfiguration struct { // features. // +optional FeatureGates map[string]bool `json:"featureGates,omitempty"` + + // The host and port that the metrics endpoint should listen on. + // The value "0" disables the metrics server. + // Defaults to '0.0.0.0:9402'. + MetricsListenAddress string `json:"metricsListenAddress,omitempty"` + + // metricsTLSConfig is used to configure the metrics server TLS settings. + MetricsTLSConfig sharedv1alpha1.TLSConfig `json:"metricsTLSConfig"` } diff --git a/pkg/apis/config/webhook/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/webhook/v1alpha1/zz_generated.deepcopy.go index e284758ae..43f50d78d 100644 --- a/pkg/apis/config/webhook/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/webhook/v1alpha1/zz_generated.deepcopy.go @@ -48,6 +48,7 @@ func (in *WebhookConfiguration) DeepCopyInto(out *WebhookConfiguration) { (*out)[key] = val } } + in.MetricsTLSConfig.DeepCopyInto(&out.MetricsTLSConfig) return } diff --git a/pkg/webhook/configfile/configfile.go b/pkg/webhook/configfile/configfile.go index 5707f6c03..c27318322 100644 --- a/pkg/webhook/configfile/configfile.go +++ b/pkg/webhook/configfile/configfile.go @@ -81,6 +81,8 @@ func WebhookConfigurationPathRefs(cfg *config.WebhookConfiguration) ([]*string, return []*string{ &cfg.TLSConfig.Filesystem.KeyFile, &cfg.TLSConfig.Filesystem.CertFile, + &cfg.MetricsTLSConfig.Filesystem.KeyFile, + &cfg.MetricsTLSConfig.Filesystem.CertFile, &cfg.KubeConfig, }, nil } diff --git a/pkg/webhook/options/options.go b/pkg/webhook/options/options.go index 95942bc15..5d1f03a67 100644 --- a/pkg/webhook/options/options.go +++ b/pkg/webhook/options/options.go @@ -90,4 +90,20 @@ func AddConfigFlags(fs *pflag.FlagSet, c *config.WebhookConfiguration) { "Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n")) logf.AddFlags(&c.Logging, fs) + + fs.StringVar(&c.MetricsListenAddress, "metrics-listen-address", c.MetricsListenAddress, "The host and port that the metrics endpoint should listen on. The value '0' disables the metrics server") + fs.StringVar(&c.MetricsTLSConfig.Filesystem.CertFile, "metrics-tls-cert-file", c.MetricsTLSConfig.Filesystem.CertFile, "path to the file containing the TLS certificate to serve metrics with") + fs.StringVar(&c.MetricsTLSConfig.Filesystem.KeyFile, "metrics-tls-private-key-file", c.MetricsTLSConfig.Filesystem.KeyFile, "path to the file containing the TLS private key to serve metrics with") + + fs.DurationVar(&c.MetricsTLSConfig.Dynamic.LeafDuration, "metrics-dynamic-serving-leaf-duration", c.MetricsTLSConfig.Dynamic.LeafDuration, "leaf duration of metrics serving certificates") + fs.StringVar(&c.MetricsTLSConfig.Dynamic.SecretNamespace, "metrics-dynamic-serving-ca-secret-namespace", c.MetricsTLSConfig.Dynamic.SecretNamespace, "namespace of the secret used to store the CA that signs metrics serving certificates") + fs.StringVar(&c.MetricsTLSConfig.Dynamic.SecretName, "metrics-dynamic-serving-ca-secret-name", c.MetricsTLSConfig.Dynamic.SecretName, "name of the secret used to store the CA that signs serving certificates") + fs.StringSliceVar(&c.MetricsTLSConfig.Dynamic.DNSNames, "metrics-dynamic-serving-dns-names", c.MetricsTLSConfig.Dynamic.DNSNames, "DNS names that should be present on certificates generated by the metrics dynamic serving CA") + fs.StringSliceVar(&c.MetricsTLSConfig.CipherSuites, "metrics-tls-cipher-suites", c.MetricsTLSConfig.CipherSuites, + "Comma-separated list of cipher suites for the metrics server. "+ + "If omitted, the default Go cipher suites will be used. "+ + "Possible values: "+strings.Join(tlsCipherPossibleValues, ",")) + fs.StringVar(&c.MetricsTLSConfig.MinTLSVersion, "metrics-tls-min-version", c.MetricsTLSConfig.MinTLSVersion, + "Minimum TLS version supported by the metrics server. If omitted, the default Go minimum version will be used. "+ + "Possible values: "+strings.Join(tlsPossibleVersions, ", ")) } diff --git a/pkg/webhook/server/server.go b/pkg/webhook/server/server.go index bd6bcd7bb..2e48786d9 100644 --- a/pkg/webhook/server/server.go +++ b/pkg/webhook/server/server.go @@ -86,6 +86,21 @@ type Server struct { // MinTLSVersion is the minimum TLS version supported. // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). MinTLSVersion string + + // The host and port that the metrics endpoint should listen on. + MetricsListenAddress string + + // If specified, the metrics server will listen with TLS using certificates + // provided by this CertificateSource. + MetricsCertificateSource servertls.CertificateSource + + // MetricsCipherSuites is the list of allowed cipher suites for the server. + // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). + MetricsCipherSuites []string + + // MetricsMinTLSVersion is the minimum TLS version supported. + // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). + MetricsMinTLSVersion string } func (s *Server) Run(ctx context.Context) error { @@ -104,6 +119,15 @@ func (s *Server) Run(ctx context.Context) error { return err } + metricsCipherSuites, err := ciphers.TLSCipherSuites(s.MetricsCipherSuites) + if err != nil { + return err + } + metricsMinVersion, err := ciphers.TLSVersion(s.MetricsMinTLSVersion) + if err != nil { + return err + } + if s.ListenAddr == 0 { webhookPort, err := freePort() if err != nil { @@ -119,7 +143,17 @@ func (s *Server) Run(ctx context.Context) error { Scheme: s.ResourceScheme, Logger: log, LeaderElection: false, // The webhook component does not need to perform leader election - Metrics: metricsserver.Options{BindAddress: "0"}, + Metrics: metricsserver.Options{ + BindAddress: s.MetricsListenAddress, + SecureServing: s.MetricsCertificateSource != nil, + TLSOpts: []func(*tls.Config){ + func(cfg *tls.Config) { + cfg.CipherSuites = metricsCipherSuites + cfg.MinVersion = metricsMinVersion + cfg.GetCertificate = s.MetricsCertificateSource.GetCertificate + }, + }, + }, WebhookServer: webhook.NewServer(webhook.Options{ Port: int(s.ListenAddr), TLSOpts: []func(*tls.Config){ @@ -139,6 +173,12 @@ func (s *Server) Run(ctx context.Context) error { return err } + if s.MetricsCertificateSource != nil { + if err := mgr.Add(s.MetricsCertificateSource); err != nil { + return err + } + } + // if a HealthzAddr is provided, start the healthz listener if s.HealthzAddr != nil { healthzListener, err := net.Listen("tcp", fmt.Sprintf(":%d", *s.HealthzAddr)) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 948a2f649..8307f6e2f 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -100,7 +100,10 @@ func RunControlPlane(t *testing.T, ctx context.Context, optionFunctions ...RunCo } webhookOpts, stopWebhook := webhooktesting.StartWebhookServer( - t, ctx, []string{"--kubeconfig", f.Name()}, + // Disable the metrics server to avoid multiple webhook servers + // attempting to listen on metrics port 9402 when tests are running in + // parallel. + t, ctx, []string{"--kubeconfig", f.Name(), "--metrics-listen-address=0"}, ) crds := readCustomResourcesAtPath(t, *options.crdsDir)