From 36aa9e250177a92e681c9f67d6a7ceb41d9c4fab Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Thu, 5 Aug 2021 14:37:54 +0100 Subject: [PATCH 1/4] The gateway-api support is now gated behind --feature-gate=ExperimentalGatewayAPISupport=true Signed-off-by: Jake Sanders --- cmd/controller/app/BUILD.bazel | 1 + cmd/controller/app/controller.go | 43 ++++++++++++--------------- cmd/controller/app/options/options.go | 5 ++++ devel/addon/certmanager/install.sh | 4 +-- pkg/feature/features.go | 15 +++++++--- 5 files changed, 38 insertions(+), 30 deletions(-) diff --git a/cmd/controller/app/BUILD.bazel b/cmd/controller/app/BUILD.bazel index 25c6d55a2..3c9f3b986 100644 --- a/cmd/controller/app/BUILD.bazel +++ b/cmd/controller/app/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//pkg/controller/certificates/trigger:go_default_library", "//pkg/controller/clusterissuers:go_default_library", "//pkg/controller/issuers:go_default_library", + "//pkg/feature:go_default_library", "//pkg/issuer/acme:go_default_library", "//pkg/issuer/acme/dns/util:go_default_library", "//pkg/issuer/ca:go_default_library", diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go index 6bcbd38f9..6dc31badb 100644 --- a/cmd/controller/app/controller.go +++ b/cmd/controller/app/controller.go @@ -52,12 +52,13 @@ import ( intscheme "github.com/jetstack/cert-manager/pkg/client/clientset/versioned/scheme" informers "github.com/jetstack/cert-manager/pkg/client/informers/externalversions" "github.com/jetstack/cert-manager/pkg/controller" - shimgw "github.com/jetstack/cert-manager/pkg/controller/certificate-shim/gateways" "github.com/jetstack/cert-manager/pkg/controller/clusterissuers" + "github.com/jetstack/cert-manager/pkg/feature" dnsutil "github.com/jetstack/cert-manager/pkg/issuer/acme/dns/util" logf "github.com/jetstack/cert-manager/pkg/logs" "github.com/jetstack/cert-manager/pkg/metrics" "github.com/jetstack/cert-manager/pkg/util" + utilfeature "github.com/jetstack/cert-manager/pkg/util/feature" ) const controllerAgentName = "cert-manager" @@ -230,30 +231,24 @@ func buildControllerContext(ctx context.Context, opts *options.ControllerOptions return nil, nil, fmt.Errorf("error creating kubernetes client: %s", err.Error()) } - // check if the gateway API CRDs are available var gatewayAvailable bool - d := cl.Discovery() - resources, err := d.ServerResourcesForGroupVersion(gwapi.GroupVersion.String()) - switch { - case apierrors.IsNotFound(err): - gatewayAvailable = false - log.Info("the Gateway API CRDs do not seem to be present, gateway-api functionality disabled") - case err != nil: - return nil, nil, fmt.Errorf("while checking if the Gateway API CRD is installed: %s", err.Error()) - case len(resources.APIResources) == 0: - gatewayAvailable = false - log.Info("the Gateway API CRDs do not seem to be present, gateway-api functionality disabled") - default: - gatewayAvailable = true - } - - // cert-manager will try watching the Gateway resources with an exponential - // back-off, which allows the user to install the CRDs after cert-manager - // itself. Let's let the user know that the CRDs have not been found yet. - if opts.EnabledControllers().Has(shimgw.ControllerName) { - if !gatewayAvailable { - log.Info("the Gateway API CRDs do not seem to be present, but the gateway-shim controller was " + - "manually enabled. please install the CRDs.") + // Check if the Gateway API feature gate was enabled + if utilfeature.DefaultFeatureGate.Enabled(feature.ExperimentalGatewayAPISupport) { + // check if the gateway API CRDs are available. If they are not found return an error + // which will cause cert-manager to crashloopbackoff + d := cl.Discovery() + resources, err := d.ServerResourcesForGroupVersion(gwapi.GroupVersion.String()) + var GatewayAPINotAvailable = "the Gateway API CRDs do not seem to be present, but " + feature.ExperimentalGatewayAPISupport + + " is set to true. Please install the gateway-api CRDs." + switch { + case apierrors.IsNotFound(err): + return nil, nil, fmt.Errorf("%s (%w)", GatewayAPINotAvailable, err) + case err != nil: + return nil, nil, fmt.Errorf("while checking if the Gateway API CRD is installed: %w", err) + case len(resources.APIResources) == 0: + return nil, nil, fmt.Errorf("%s (found %d APIResources in %s)", GatewayAPINotAvailable, len(resources.APIResources), gwapi.GroupVersion.String()) + default: + gatewayAvailable = true } } diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go index 89f19e435..6fad64975 100644 --- a/cmd/controller/app/options/options.go +++ b/cmd/controller/app/options/options.go @@ -424,5 +424,10 @@ func (o *ControllerOptions) EnabledControllers() sets.String { enabled = enabled.Insert(experimentalCertificateSigningRequestControllers...) } + if utilfeature.DefaultFeatureGate.Enabled(feature.ExperimentalGatewayAPISupport) { + logf.Log.Info("enabling the sig-network Gateway API certificate-shim and HTTP-01 solver") + enabled = enabled.Insert(shimgatewaycontroller.ControllerName) + } + return enabled } diff --git a/devel/addon/certmanager/install.sh b/devel/addon/certmanager/install.sh index 37bfbbee4..453c528a8 100755 --- a/devel/addon/certmanager/install.sh +++ b/devel/addon/certmanager/install.sh @@ -23,7 +23,7 @@ NAMESPACE="${NAMESPACE:-cert-manager}" # Release name to use with Helm RELEASE_NAME="${RELEASE_NAME:-cert-manager}" # Default feature gates to enable -FEATURE_GATES="${FEATURE_GATES:-ExperimentalCertificateSigningRequestControllers=true}" +FEATURE_GATES="${FEATURE_GATES:-ExperimentalCertificateSigningRequestControllers=true,ExperimentalGatewayAPISupport=true}" SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") source "${SCRIPT_ROOT}/../../lib/lib.sh" @@ -66,7 +66,7 @@ helm upgrade \ --set startupapicheck.image.tag="${APP_VERSION}" \ --set installCRDs=true \ --set featureGates="${FEATURE_GATES:-}" \ - --set "extraArgs={--dns01-recursive-nameservers=${SERVICE_IP_PREFIX}.16:53,--dns01-recursive-nameservers-only=true,--controllers=*\,gateway-shim}" \ + --set "extraArgs={--dns01-recursive-nameservers=${SERVICE_IP_PREFIX}.16:53,--dns01-recursive-nameservers-only=true}" \ "$RELEASE_NAME" \ "$REPO_ROOT/bazel-bin/deploy/charts/cert-manager/cert-manager.tgz" diff --git a/pkg/feature/features.go b/pkg/feature/features.go index 7acb5dd52..07274ab4e 100644 --- a/pkg/feature/features.go +++ b/pkg/feature/features.go @@ -34,16 +34,23 @@ const ( // ExperimentalCertificateSigningRequestControllers enables all CertificateSigningRequest // controllers that sign Kubernetes CertificateSigningRequest resources ExperimentalCertificateSigningRequestControllers featuregate.Feature = "ExperimentalCertificateSigningRequestControllers" + + // alpha: v1.5.0 + // + // ExperimentalGatewayAPISupport enables the gateway-shim controller and adds support for + // the Gateway API to the HTTP-01 challenge solver. + ExperimentalGatewayAPISupport featuregate.Feature = "ExperimentalGatewayAPISupport" ) func init() { - runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)) + runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultCertManagerFeatureGates)) } -// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys. +// defaultCertManagerFeatureGates consists of all known cert-manager feature keys. // To add a new feature, define a key for it above and add it here. The features will be -// available throughout Kubernetes binaries. -var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ +// available on the cert-manager controller binary. +var defaultCertManagerFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ ValidateCAA: {Default: false, PreRelease: featuregate.Alpha}, ExperimentalCertificateSigningRequestControllers: {Default: false, PreRelease: featuregate.Alpha}, + ExperimentalGatewayAPISupport: {Default: false, PreRelease: featuregate.Alpha}, } From d4c8aea472916ca4a652f2a5dac9e9bfe30864a1 Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Thu, 5 Aug 2021 15:12:41 +0100 Subject: [PATCH 2/4] Helm --set requires commas to be escaped, so double escape in bash Signed-off-by: Jake Sanders --- devel/addon/certmanager/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devel/addon/certmanager/install.sh b/devel/addon/certmanager/install.sh index 453c528a8..7536da91d 100755 --- a/devel/addon/certmanager/install.sh +++ b/devel/addon/certmanager/install.sh @@ -23,7 +23,7 @@ NAMESPACE="${NAMESPACE:-cert-manager}" # Release name to use with Helm RELEASE_NAME="${RELEASE_NAME:-cert-manager}" # Default feature gates to enable -FEATURE_GATES="${FEATURE_GATES:-ExperimentalCertificateSigningRequestControllers=true,ExperimentalGatewayAPISupport=true}" +FEATURE_GATES="${FEATURE_GATES:-ExperimentalCertificateSigningRequestControllers=true\\,ExperimentalGatewayAPISupport=true}" SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") source "${SCRIPT_ROOT}/../../lib/lib.sh" From ed4ad50b226987bc584ac47579f67ccf136b3f59 Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Thu, 5 Aug 2021 15:22:02 +0100 Subject: [PATCH 3/4] Don't start the Gateway Shared Informer Factory if the Gateway API feature is disabled Signed-off-by: Jake Sanders --- cmd/controller/app/controller.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go index 6dc31badb..4b1d97205 100644 --- a/cmd/controller/app/controller.go +++ b/cmd/controller/app/controller.go @@ -194,7 +194,10 @@ func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { log.V(logf.DebugLevel).Info("starting shared informer factories") ctx.SharedInformerFactory.Start(rootCtx.Done()) ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) - ctx.GWShared.Start(rootCtx.Done()) + + if utilfeature.DefaultFeatureGate.Enabled(feature.ExperimentalGatewayAPISupport) { + ctx.GWShared.Start(rootCtx.Done()) + } err = g.Wait() if err != nil { From b4af1aff1af734c93893683a2a9fa6d6d65b4322 Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Fri, 6 Aug 2021 11:44:10 +0100 Subject: [PATCH 4/4] Never send an empty argument to the test script Signed-off-by: Jake Sanders --- devel/run-e2e.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devel/run-e2e.sh b/devel/run-e2e.sh index e439e886b..2cd76e317 100755 --- a/devel/run-e2e.sh +++ b/devel/run-e2e.sh @@ -44,7 +44,7 @@ bazel build //test/e2e:e2e.test echo "Using K8S_VERSION ${K8S_VERSION}" case "$K8S_VERSION" in "1.16" | "1.17" | "1.18") - SKIP="--ginkgo.skip=Gateway" + SKIP="Gateway" echo "skipping Gateway e2e tests as K8S_VERSION is <1.19" ;; *) @@ -59,5 +59,5 @@ ginkgo -nodes 10 -flakeAttempts ${FLAKE_ATTEMPTS:-1} \ --report-dir="${ARTIFACTS:-$REPO_ROOT/_artifacts}" \ --acme-dns-server="$DNS_SERVER" \ --acme-ingress-ip="$INGRESS_IP" \ - "${SKIP}" \ + "--ginkgo.skip=${SKIP}" \ "$@"