diff --git a/cmd/ctl/BUILD.bazel b/cmd/ctl/BUILD.bazel index 01d55a0d5..373b66152 100644 --- a/cmd/ctl/BUILD.bazel +++ b/cmd/ctl/BUILD.bazel @@ -61,6 +61,7 @@ filegroup( "//cmd/ctl/pkg/install:all-srcs", "//cmd/ctl/pkg/renew:all-srcs", "//cmd/ctl/pkg/status:all-srcs", + "//cmd/ctl/pkg/upgrade:all-srcs", "//cmd/ctl/pkg/version:all-srcs", ], tags = ["automanaged"], diff --git a/cmd/ctl/pkg/build/commands/BUILD.bazel b/cmd/ctl/pkg/build/commands/BUILD.bazel index b22d8ea9a..ebea90219 100644 --- a/cmd/ctl/pkg/build/commands/BUILD.bazel +++ b/cmd/ctl/pkg/build/commands/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//cmd/ctl/pkg/inspect:go_default_library", "//cmd/ctl/pkg/renew:go_default_library", "//cmd/ctl/pkg/status:go_default_library", + "//cmd/ctl/pkg/upgrade:go_default_library", "//cmd/ctl/pkg/version:go_default_library", "@com_github_spf13_cobra//:go_default_library", "@io_k8s_cli_runtime//pkg/genericclioptions:go_default_library", diff --git a/cmd/ctl/pkg/build/commands/commands.go b/cmd/ctl/pkg/build/commands/commands.go index 3319c6046..bd723c995 100644 --- a/cmd/ctl/pkg/build/commands/commands.go +++ b/cmd/ctl/pkg/build/commands/commands.go @@ -33,6 +33,7 @@ import ( "github.com/jetstack/cert-manager/cmd/ctl/pkg/inspect" "github.com/jetstack/cert-manager/cmd/ctl/pkg/renew" "github.com/jetstack/cert-manager/cmd/ctl/pkg/status" + "github.com/jetstack/cert-manager/cmd/ctl/pkg/upgrade" "github.com/jetstack/cert-manager/cmd/ctl/pkg/version" ) @@ -56,6 +57,7 @@ func Commands() []RegisterCommandFunc { approve.NewCmdApprove, deny.NewCmdDeny, check.NewCmdCheck, + upgrade.NewCmdUpgrade, // Experimental features experimental.NewCmdExperimental, diff --git a/cmd/ctl/pkg/upgrade/BUILD.bazel b/cmd/ctl/pkg/upgrade/BUILD.bazel new file mode 100644 index 000000000..698fb462c --- /dev/null +++ b/cmd/ctl/pkg/upgrade/BUILD.bazel @@ -0,0 +1,30 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["upgrade.go"], + importpath = "github.com/jetstack/cert-manager/cmd/ctl/pkg/upgrade", + visibility = ["//visibility:public"], + deps = [ + "//cmd/ctl/pkg/upgrade/migrateapiversion:go_default_library", + "@com_github_spf13_cobra//:go_default_library", + "@io_k8s_cli_runtime//pkg/genericclioptions:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//cmd/ctl/pkg/upgrade/migrateapiversion:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/ctl/pkg/upgrade/migrateapiversion/BUILD.bazel b/cmd/ctl/pkg/upgrade/migrateapiversion/BUILD.bazel new file mode 100644 index 000000000..83e382363 --- /dev/null +++ b/cmd/ctl/pkg/upgrade/migrateapiversion/BUILD.bazel @@ -0,0 +1,46 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "command.go", + "migrator.go", + ], + importpath = "github.com/jetstack/cert-manager/cmd/ctl/pkg/upgrade/migrateapiversion", + visibility = ["//visibility:public"], + deps = [ + "//cmd/ctl/pkg/build:go_default_library", + "//cmd/ctl/pkg/factory:go_default_library", + "//internal/apis/acme/install:go_default_library", + "//internal/apis/certmanager/install:go_default_library", + "@com_github_spf13_cobra//:go_default_library", + "@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/install:go_default_library", + "@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/errors:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured:go_default_library", + "@io_k8s_apimachinery//pkg/runtime:go_default_library", + "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library", + "@io_k8s_apimachinery//pkg/util/sets:go_default_library", + "@io_k8s_apimachinery//pkg/util/wait:go_default_library", + "@io_k8s_cli_runtime//pkg/genericclioptions:go_default_library", + "@io_k8s_client_go//util/retry:go_default_library", + "@io_k8s_kubectl//pkg/cmd/util:go_default_library", + "@io_k8s_kubectl//pkg/util/i18n:go_default_library", + "@io_k8s_kubectl//pkg/util/templates:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/client:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/ctl/pkg/upgrade/migrateapiversion/command.go b/cmd/ctl/pkg/upgrade/migrateapiversion/command.go new file mode 100644 index 000000000..9d4a14691 --- /dev/null +++ b/cmd/ctl/pkg/upgrade/migrateapiversion/command.go @@ -0,0 +1,140 @@ +/* +Copyright 2022 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrateapiversion + +import ( + "context" + + "github.com/spf13/cobra" + apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/jetstack/cert-manager/cmd/ctl/pkg/build" + "github.com/jetstack/cert-manager/cmd/ctl/pkg/factory" + acmeinstall "github.com/jetstack/cert-manager/internal/apis/acme/install" + cminstall "github.com/jetstack/cert-manager/internal/apis/certmanager/install" +) + +var ( + long = templates.LongDesc(i18n.T(` +Ensures resources in your Kubernetes cluster are persisted in the v1 API version. + +This must be run prior to upgrading to ensure your cluster is ready to upgrade to cert-manager v1.7 and beyond. + +This command must be run with a cluster running cert-manager v1.0 or greater.`)) + + example = templates.Examples(i18n.T(build.WithTemplate(` +# Check the cert-manager installation is ready to be upgraded to v1.7 and perform necessary migrations +# to ensure that the kube-apiserver has stored only v1 API versions. +{{.BuildName}} upgrade migrate-api-version + +# Force migrations to be run, even if the 'status.storedVersion' field on the CRDs does not contain +# old, deprecated API versions. +# This should only be used if you have manually edited/patched the CRDs already. +# It will force a read and a write of ALL cert-manager resources unconditionally. +{{.BuildName}} upgrade migrate-api-version --skip-stored-version-check +`))) +) + +// Options is a struct to support renew command +type Options struct { + genericclioptions.IOStreams + *factory.Factory + + client client.Client + skipStoredVersionCheck bool + qps float32 + burst int +} + +// NewOptions returns initialized Options +func NewOptions(ioStreams genericclioptions.IOStreams) *Options { + return &Options{ + IOStreams: ioStreams, + } +} + +// NewCmdMigrate returns a cobra command for updating resources in an apiserver +// to force a new storage version to be used. +func NewCmdMigrate(ctx context.Context, ioStreams genericclioptions.IOStreams) *cobra.Command { + o := NewOptions(ioStreams) + cmd := &cobra.Command{ + Use: "migrate-api-version", + Short: "Migrate all existing persisted cert-manager resources to the v1 API version", + Long: long, + Example: example, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.Validate(args)) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Run(ctx, args)) + }, + } + + cmd.Flags().BoolVar(&o.skipStoredVersionCheck, "skip-stored-version-check", o.skipStoredVersionCheck, ""+ + "If true, all resources will be read and written regardless of the 'status.storedVersions' on the CRD resource. "+ + "Use this mode if you have previously manually modified the 'status.storedVersions' field on CRD resources.") + cmd.Flags().Float32Var(&o.qps, "qps", 5, "Indicates the maximum QPS to the apiserver from the client.") + cmd.Flags().IntVar(&o.burst, "burst", 10, "Maximum burst value for queries set to the apiserver from the client.") + o.Factory = factory.New(ctx, cmd) + + return cmd +} + +// Validate validates the provided options +func (o *Options) Validate(_ []string) error { + return nil +} + +// Complete takes the command arguments and factory and infers any remaining options. +func (o *Options) Complete() error { + var err error + scheme := runtime.NewScheme() + apiextinstall.Install(scheme) + cminstall.Install(scheme) + acmeinstall.Install(scheme) + + if o.qps != 0 { + o.RESTConfig.QPS = o.qps + } + if o.burst != 0 { + o.RESTConfig.Burst = o.burst + } + o.client, err = client.New(o.RESTConfig, client.Options{Scheme: scheme}) + if err != nil { + return err + } + + return nil +} + +// Run executes renew command +func (o *Options) Run(ctx context.Context, args []string) error { + _, err := NewMigrator(o.client, o.skipStoredVersionCheck, o.Out, o.ErrOut).Run(ctx, "v1", []string{ + "certificates.cert-manager.io", + "certificaterequests.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + }) + return err +} diff --git a/cmd/ctl/pkg/upgrade/migrateapiversion/migrator.go b/cmd/ctl/pkg/upgrade/migrateapiversion/migrator.go new file mode 100644 index 000000000..50869efb8 --- /dev/null +++ b/cmd/ctl/pkg/upgrade/migrateapiversion/migrator.go @@ -0,0 +1,281 @@ +/* +Copyright 2022 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrateapiversion + +import ( + "context" + "fmt" + "io" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + + apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Migrator struct { + // Client used for API interactions + Client client.Client + + // If true, skip checking the 'status.storedVersion' before running the migration. + // By default, migration will only be run if the CRD contains storedVersions other + // than the desired target version. + SkipStoredVersionCheck bool + + // Writers to write informational & error messages to + Out, ErrOut io.Writer +} + +// NewMigrator creates a new migrator with the given API client. +// If either of out or errOut are nil, log messages will be discarded. +func NewMigrator(client client.Client, skipStoredVersionCheck bool, out, errOut io.Writer) *Migrator { + if out == nil { + out = io.Discard + } + if errOut == nil { + errOut = io.Discard + } + + return &Migrator{ + Client: client, + SkipStoredVersionCheck: skipStoredVersionCheck, + Out: out, + ErrOut: errOut, + } +} + +// Run begins the migration of all the named CRDs. +// It will attempt to migrate all resources defined as part of these CRDs to the +// given 'targetVersion', and after completion will update the `status.storedVersions` +// field on the corresponding CRD version to only contain the given targetVersion. +// Returns 'true' if a migration was actually performed, and false if migration was not required. +func (m *Migrator) Run(ctx context.Context, targetVersion string, names []string) (bool, error) { + fmt.Fprintf(m.Out, "Checking all CustomResourceDefinitions have storage version set to '%s'\n", targetVersion) + allTargetVersion, allCRDs, err := m.ensureCRDStorageVersionEquals(ctx, targetVersion, names) + if err != nil { + return false, err + } + if !allTargetVersion { + fmt.Fprintf(m.ErrOut, "It looks like you are running a version of cert-manager that does not set the storage version of CRDs to %q. You MUST upgrade to cert-manager v1.0-v1.6 before migrating resources for v1.7.\n", targetVersion) + return false, fmt.Errorf("preflight checks failed") + } + fmt.Fprintf(m.Out, "All CustomResourceDefinitions have %q configured as the storage version.\n", targetVersion) + + crdsRequiringMigration := allCRDs + if !m.SkipStoredVersionCheck { + fmt.Fprintf(m.Out, "Looking for CRDs that contain resources that require migrating to %q...\n", targetVersion) + crdsRequiringMigration, err = m.discoverCRDsRequiringMigration(ctx, targetVersion, names) + if err != nil { + fmt.Fprintf(m.ErrOut, "Failed to determine resource types that require migration: %v\n", err) + return false, err + } + if len(crdsRequiringMigration) == 0 { + fmt.Fprintln(m.Out, "Nothing to do. cert-manager CRDs do not have 'status.storedVersions' containing old API versions. You may proceed to upgrade to cert-manager v1.7.") + return false, nil + } + } else { + fmt.Fprintln(m.Out, "Forcing migration of all CRD resources as --skip-stored-version-check=true") + } + + fmt.Fprintf(m.Out, "Found %d resource types that require migration:\n", len(crdsRequiringMigration)) + for _, crd := range crdsRequiringMigration { + fmt.Fprintf(m.Out, " - %s\n", crd.Name) + } + + for _, crd := range crdsRequiringMigration { + if err := m.migrateResourcesForCRD(ctx, crd); err != nil { + fmt.Fprintf(m.ErrOut, "Failed to migrate resource: %v\n", err) + return false, err + } + } + + fmt.Fprintf(m.Out, "Patching CRD resources to set 'status.storedVersions' to %q...\n", targetVersion) + if err := m.patchCRDStoredVersions(ctx, crdsRequiringMigration); err != nil { + fmt.Fprintf(m.ErrOut, "Failed to patch 'status.storedVersions' field: %v\n", err) + return false, err + } + + fmt.Fprintln(m.Out, "Successfully migrated all cert-manager resource types. It is now safe to proceed with upgrading to cert-manager v1.7.") + return true, nil +} + +func (m *Migrator) ensureCRDStorageVersionEquals(ctx context.Context, vers string, names []string) (bool, []*apiext.CustomResourceDefinition, error) { + var crds []*apiext.CustomResourceDefinition + for _, crdName := range names { + crd := &apiext.CustomResourceDefinition{} + if err := m.Client.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil { + return false, nil, err + } + + // Discover the storage version + storageVersion := storageVersionForCRD(crd) + + if storageVersion != vers { + fmt.Fprintf(m.Out, "CustomResourceDefinition object %q has storage version set to %q.\n", crdName, storageVersion) + return false, nil, nil + } + + crds = append(crds, crd) + } + + return true, crds, nil +} + +func (m *Migrator) discoverCRDsRequiringMigration(ctx context.Context, desiredStorageVersion string, names []string) ([]*apiext.CustomResourceDefinition, error) { + var requireMigration []*apiext.CustomResourceDefinition + for _, name := range names { + crd := &apiext.CustomResourceDefinition{} + if err := m.Client.Get(ctx, client.ObjectKey{Name: name}, crd); err != nil { + return nil, err + } + // If no versions are stored, there's nothing to migrate. + if len(crd.Status.StoredVersions) == 0 { + continue + } + // If more than one entry exists in `storedVersions` OR if the only element in there is not + // the desired version, perform a migration. + if len(crd.Status.StoredVersions) > 1 || crd.Status.StoredVersions[0] != desiredStorageVersion { + requireMigration = append(requireMigration, crd) + } + } + return requireMigration, nil +} + +func (m *Migrator) migrateResourcesForCRD(ctx context.Context, crd *apiext.CustomResourceDefinition) error { + startTime := time.Now() + fmt.Fprintf(m.Out, "Migrating %q objects in group %q - this may take a while (started at %s)...\n", crd.Spec.Names.Kind, crd.Spec.Group, startTime.Format(time.Stamp)) + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: storageVersionForCRD(crd), + Kind: crd.Spec.Names.ListKind, + }) + if err := m.Client.List(ctx, list); err != nil { + return err + } + fmt.Fprintf(m.Out, " %d resources to migrate\n", len(list.Items)) + for _, obj := range list.Items { + // retry on any kind of error to handle cases where e.g. the network connection to the apiserver fails + if err := retry.OnError(wait.Backoff{ + Duration: time.Second, // wait 1s between attempts + Steps: 3, // allow up to 3 attempts per object + }, func(err error) bool { + // Retry on any errors that are not otherwise skipped/ignored + return handleUpdateErr(err) != nil + }, func() error { return m.Client.Update(ctx, &obj) }); handleUpdateErr(err) != nil { + return err + } + } + fmt.Fprintf(m.Out, " Successfully migrated %d %s objects in %s\n", len(list.Items), crd.Spec.Names.Kind, time.Now().Sub(startTime).Round(time.Second)) + return nil +} + +// patchCRDStoredVersions will patch the `status.storedVersions` field of all passed in CRDs to be +// set to an array containing JUST the current storage version. +// This is only safe to run after a successful migration (i.e. a read/write of all resources of the given CRD type). +func (m *Migrator) patchCRDStoredVersions(ctx context.Context, crds []*apiext.CustomResourceDefinition) error { + for _, crd := range crds { + // fetch a fresh copy of the CRD to avoid any conflict errors + freshCRD := &apiext.CustomResourceDefinition{} + if err := m.Client.Get(ctx, client.ObjectKey{Name: crd.Name}, freshCRD); err != nil { + return err + } + + // Check the latest copy of the CRD to ensure that: + // 1) the storage version is the same as it was at the start of the migration + // 2) the status.storedVersion field has not changed, and if it has, it has only added the new/desired storage version + // This helps to avoid cases where the storage version was changed by a third-party midway through the migration, + // which could lead to corrupted apiservers when we patch the status.storedVersions field below. + expectedStorageVersion := storageVersionForCRD(crd) + if storageVersionForCRD(freshCRD) != expectedStorageVersion { + return newUnexpectedChangeError(crd) + } + newlyAddedVersions := storedVersionsAdded(crd, freshCRD) + if newlyAddedVersions.Len() != 0 && !newlyAddedVersions.Equal(sets.NewString(expectedStorageVersion)) { + return newUnexpectedChangeError(crd) + } + + // Set the `status.storedVersions` field to the target storage version + freshCRD.Status.StoredVersions = []string{storageVersionForCRD(crd)} + + if err := m.Client.Status().Update(ctx, freshCRD); err != nil { + return err + } + } + + return nil +} + +// storageVersionForCRD discovers the storage version for a given CRD. +func storageVersionForCRD(crd *apiext.CustomResourceDefinition) string { + storageVersion := "" + for _, v := range crd.Spec.Versions { + if v.Storage { + storageVersion = v.Name + break + } + } + return storageVersion +} + +// storedVersionsAdded returns a list of any versions added to the `status.storedVersions` field on +// a CRD resource. +func storedVersionsAdded(old, new *apiext.CustomResourceDefinition) sets.String { + oldStoredVersions := sets.NewString(old.Status.StoredVersions...) + newStoredVersions := sets.NewString(new.Status.StoredVersions...) + return newStoredVersions.Difference(oldStoredVersions) +} + +// newUnexpectedChangeError creates a new 'error' that informs users that a change to the CRDs +// was detected during the migration process and so the migration must be re-run. +func newUnexpectedChangeError(crd *apiext.CustomResourceDefinition) error { + errorFmt := "" + + "The CRD %q unexpectedly changed during the migration. " + + "This means that either an object was persisted in a non-storage version during the migration, " + + "or the storage version was changed by someone else (or some automated deployment tooling) whilst the migration " + + "was in progress.\n\n" + + "All automated deployment tooling should be in a 'stable state' (i.e. no upgrades to cert-manager CRDs should be" + + "in progress whilst the migration is running).\n\n" + + "Please ensure no changes to the CRDs are made during the migration process and re-run the migration until you" + + "no longer see this message." + return fmt.Errorf(errorFmt, crd.Name) +} + +// handleUpdateErr will absorb certain types of errors that we know can be skipped/passed on +// during a migration of a particular object. +func handleUpdateErr(err error) error { + if err == nil { + return nil + } + // If the resource no longer exists, don't return the error as the object no longer + // needs updating to the new API version. + if apierrors.IsNotFound(err) { + return nil + } + // If there was a conflict, another client must have written the object already which + // means we don't need to force an update. + if apierrors.IsConflict(err) { + return nil + } + return err +} diff --git a/cmd/ctl/pkg/upgrade/upgrade.go b/cmd/ctl/pkg/upgrade/upgrade.go new file mode 100644 index 000000000..2688da38d --- /dev/null +++ b/cmd/ctl/pkg/upgrade/upgrade.go @@ -0,0 +1,38 @@ +/* +Copyright 2022 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "context" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + + "github.com/jetstack/cert-manager/cmd/ctl/pkg/upgrade/migrateapiversion" +) + +func NewCmdUpgrade(ctx context.Context, ioStreams genericclioptions.IOStreams) *cobra.Command { + cmds := &cobra.Command{ + Use: "upgrade", + Short: "Tools that assist in upgrading cert-manager", + Long: `Note: this command does NOT actually upgrade cert-manager installations`, + } + + cmds.AddCommand(migrateapiversion.NewCmdMigrate(ctx, ioStreams)) + + return cmds +} diff --git a/test/integration/ctl/BUILD.bazel b/test/integration/ctl/BUILD.bazel index ced010a46..c0a0d0547 100644 --- a/test/integration/ctl/BUILD.bazel +++ b/test/integration/ctl/BUILD.bazel @@ -49,6 +49,7 @@ filegroup( srcs = [ ":package-srcs", "//test/integration/ctl/install_framework:all-srcs", + "//test/integration/ctl/migrate:all-srcs", ], tags = ["automanaged"], visibility = ["//visibility:public"], diff --git a/test/integration/ctl/migrate/BUILD.bazel b/test/integration/ctl/migrate/BUILD.bazel new file mode 100644 index 000000000..e26348168 --- /dev/null +++ b/test/integration/ctl/migrate/BUILD.bazel @@ -0,0 +1,39 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "go_default_test", + srcs = ["ctl_upgrade_migrate_test.go"], + data = [ + "//pkg/webhook/handlers/testdata/apis/testgroup/crds:all-srcs", + ], + deps = [ + "//cmd/ctl/pkg/upgrade/migrateapiversion:go_default_library", + "//pkg/webhook/handlers:go_default_library", + "//pkg/webhook/handlers/testdata/apis/testgroup/install:go_default_library", + "//pkg/webhook/handlers/testdata/apis/testgroup/v1:go_default_library", + "//pkg/webhook/handlers/testdata/apis/testgroup/v2:go_default_library", + "//test/integration/framework:go_default_library", + "@com_github_go_logr_logr//testing:go_default_library", + "@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/install:go_default_library", + "@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured:go_default_library", + "@io_k8s_apimachinery//pkg/runtime:go_default_library", + "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/client:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/integration/ctl/migrate/ctl_upgrade_migrate_test.go b/test/integration/ctl/migrate/ctl_upgrade_migrate_test.go new file mode 100644 index 000000000..2144adf00 --- /dev/null +++ b/test/integration/ctl/migrate/ctl_upgrade_migrate_test.go @@ -0,0 +1,384 @@ +/* +Copyright 2022 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + "context" + "os" + "testing" + "time" + + testlogger "github.com/go-logr/logr/testing" + apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/jetstack/cert-manager/cmd/ctl/pkg/upgrade/migrateapiversion" + "github.com/jetstack/cert-manager/pkg/webhook/handlers" + "github.com/jetstack/cert-manager/pkg/webhook/handlers/testdata/apis/testgroup/install" + "github.com/jetstack/cert-manager/pkg/webhook/handlers/testdata/apis/testgroup/v1" + "github.com/jetstack/cert-manager/pkg/webhook/handlers/testdata/apis/testgroup/v2" + "github.com/jetstack/cert-manager/test/integration/framework" +) + +// Create a test resource at a given version. +func newResourceAtVersion(t *testing.T, version string) client.Object { + switch version { + case "v1": + return &v1.TestType{ + ObjectMeta: metav1.ObjectMeta{ + Name: "object", + Namespace: "default", + }, + TestField: "abc", + TestFieldImmutable: "def", + } + case "v2": + return &v2.TestType{ + ObjectMeta: metav1.ObjectMeta{ + Name: "object", + Namespace: "default", + }, + TestField: "abc", + TestFieldImmutable: "def", + } + default: + t.Fatalf("unknown version %q", version) + } + return nil +} + +func newScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + apiextinstall.Install(scheme) + install.Install(scheme) + return scheme +} + +func TestCtlUpgradeMigrate(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + // Create the control plane with the TestType conversion handlers registered + scheme := newScheme() + // name of the testtype CRD resource + crdName := "testtypes.testgroup.testing.cert-manager.io" + restCfg, stop := framework.RunControlPlane(t, context.Background(), + framework.WithCRDDirectory("../../../../pkg/webhook/handlers/testdata/apis/testgroup/crds"), + framework.WithWebhookConversionHandler(handlers.NewSchemeBackedConverter(testlogger.NewTestLogger(t), scheme))) + defer stop() + + // Ensure the OpenAPI endpoint has been updated with the TestType CRD + framework.WaitForOpenAPIResourcesToBeLoaded(t, ctx, restCfg, schema.GroupVersionKind{ + Group: "testgroup.testing.cert-manager.io", + Version: "v1", + Kind: "TestType", + }) + + // Create an API client + cl, err := client.New(restCfg, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + + // Fetch a copy of the recently created TestType CRD + crd := &apiext.CustomResourceDefinition{} + if err := cl.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil { + t.Fatal(err) + } + + // Identify the current storage version and one non-storage version for this CRD. + // We'll be creating objects and then changing the storage version on the CRD to + // the 'nonStorageVersion' and ensuring the migration/upgrade is successful. + storageVersion, nonStorageVersion := versionsForCRD(crd) + if storageVersion == "" || nonStorageVersion == "" { + t.Fatal("this test requires testdata with both a storage and non-storage version set") + } + + // Ensure the original storage version is the only one on the CRD + if len(crd.Status.StoredVersions) != 1 || crd.Status.StoredVersions[0] != storageVersion { + t.Errorf("Expected status.storedVersions to only contain the storage version %q but it was: %v", storageVersion, crd.Status.StoredVersions) + } + + // Create a resource + obj := newResourceAtVersion(t, storageVersion) + if err := cl.Create(ctx, obj); err != nil { + t.Errorf("Failed to create test resource: %v", err) + } + + // Set the storage version to the 'nonStorageVersion' + setStorageVersion(crd, nonStorageVersion) + if err := cl.Update(ctx, crd); err != nil { + t.Fatalf("Failed to update CRD storage version: %v", err) + } + if len(crd.Status.StoredVersions) != 2 || crd.Status.StoredVersions[0] != storageVersion || crd.Status.StoredVersions[1] != nonStorageVersion { + t.Fatalf("Expected status.storedVersions to contain [%s, %s] but it was: %v", storageVersion, nonStorageVersion, crd.Status.StoredVersions) + } + + // Run the migrator and migrate all objects to the 'nonStorageVersion' (which is now the new storage version) + migrator := migrateapiversion.NewMigrator(cl, false, os.Stdout, os.Stderr) + migrated, err := migrator.Run(ctx, nonStorageVersion, []string{crdName}) + if err != nil { + t.Errorf("migrator failed to run: %v", err) + } + if !migrated { + t.Errorf("migrator didn't actually perform a migration") + } + + // Check the status.storedVersions field to ensure it only contains one element + crd = &apiext.CustomResourceDefinition{} + if err := cl.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil { + t.Fatal(err) + } + if len(crd.Status.StoredVersions) != 1 || crd.Status.StoredVersions[0] != nonStorageVersion { + t.Fatalf("Expected status.storedVersions to be %q but it was: %v", nonStorageVersion, crd.Status.StoredVersions) + } + + // Remove the previous storage version from the CRD and update it + removeAPIVersion(crd, storageVersion) + if err := cl.Update(ctx, crd); err != nil { + t.Fatalf("Failed to remove old API version: %v", err) + } + + // Attempt to read a resource list in the new API version + objList := &unstructured.UnstructuredList{} + objList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: nonStorageVersion, + Kind: crd.Spec.Names.ListKind, + }) + if err := cl.List(ctx, objList); err != nil { + t.Fatalf("Failed to list objects (gvk %v): %v", objList.GroupVersionKind(), err) + } + if len(objList.Items) != 1 { + t.Fatalf("Expected a single TestType resource to exist") + } +} + +func TestCtlUpgradeMigrate_FailsIfStorageVersionDoesNotEqualTargetVersion(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + // Create the control plane with the TestType conversion handlers registered + scheme := newScheme() + // name of the testtype CRD resource + crdName := "testtypes.testgroup.testing.cert-manager.io" + restCfg, stop := framework.RunControlPlane(t, context.Background(), + framework.WithCRDDirectory("../../../../pkg/webhook/handlers/testdata/apis/testgroup/crds"), + framework.WithWebhookConversionHandler(handlers.NewSchemeBackedConverter(testlogger.NewTestLogger(t), scheme))) + defer stop() + + // Ensure the OpenAPI endpoint has been updated with the TestType CRD + framework.WaitForOpenAPIResourcesToBeLoaded(t, ctx, restCfg, schema.GroupVersionKind{ + Group: "testgroup.testing.cert-manager.io", + Version: "v1", + Kind: "TestType", + }) + + // Create an API client + cl, err := client.New(restCfg, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + + // Fetch a copy of the recently created TestType CRD + crd := &apiext.CustomResourceDefinition{} + if err := cl.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil { + t.Fatal(err) + } + + // Identify the current storage version and one non-storage version for this CRD. + storageVersion, nonStorageVersion := versionsForCRD(crd) + if storageVersion == "" || nonStorageVersion == "" { + t.Fatal("this test requires testdata with both a storage and non-storage version set") + } + + // We expect this to fail, as we are attempting to migrate to the 'nonStorageVersion'. + migrator := migrateapiversion.NewMigrator(cl, false, os.Stdout, os.Stderr) + migrated, err := migrator.Run(ctx, nonStorageVersion, []string{crdName}) + if err == nil { + t.Errorf("expected an error to be returned but we got none") + } + if err.Error() != "preflight checks failed" { + t.Errorf("unexpected error: %v", err) + } + if migrated { + t.Errorf("migrator ran but it should not have") + } +} + +func TestCtlUpgradeMigrate_SkipsMigrationIfNothingToDo(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + // Create the control plane with the TestType conversion handlers registered + scheme := newScheme() + // name of the testtype CRD resource + crdName := "testtypes.testgroup.testing.cert-manager.io" + restCfg, stop := framework.RunControlPlane(t, context.Background(), + framework.WithCRDDirectory("../../../../pkg/webhook/handlers/testdata/apis/testgroup/crds"), + framework.WithWebhookConversionHandler(handlers.NewSchemeBackedConverter(testlogger.NewTestLogger(t), scheme))) + defer stop() + + // Ensure the OpenAPI endpoint has been updated with the TestType CRD + framework.WaitForOpenAPIResourcesToBeLoaded(t, ctx, restCfg, schema.GroupVersionKind{ + Group: "testgroup.testing.cert-manager.io", + Version: "v1", + Kind: "TestType", + }) + + // Create an API client + cl, err := client.New(restCfg, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + + // Fetch a copy of the recently created TestType CRD + crd := &apiext.CustomResourceDefinition{} + if err := cl.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil { + t.Fatal(err) + } + + // Identify the current storage version and one non-storage version for this CRD. + storageVersion, nonStorageVersion := versionsForCRD(crd) + if storageVersion == "" || nonStorageVersion == "" { + t.Fatal("this test requires testdata with both a storage and non-storage version set") + } + + // Ensure the original storage version is the only one on the CRD + if len(crd.Status.StoredVersions) != 1 || crd.Status.StoredVersions[0] != storageVersion { + t.Errorf("Expected status.storedVersions to only contain the storage version %q but it was: %v", storageVersion, crd.Status.StoredVersions) + } + + // Create a resource + obj := newResourceAtVersion(t, storageVersion) + if err := cl.Create(ctx, obj); err != nil { + t.Errorf("Failed to create test resource: %v", err) + } + + // We expect this to succeed and for the migration to not be run + migrator := migrateapiversion.NewMigrator(cl, false, os.Stdout, os.Stderr) + migrated, err := migrator.Run(ctx, storageVersion, []string{crdName}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if migrated { + t.Errorf("migrator ran but it should not have") + } +} + +func TestCtlUpgradeMigrate_ForcesMigrationIfSkipStoredVersionCheckIsEnabled(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + // Create the control plane with the TestType conversion handlers registered + scheme := newScheme() + // name of the testtype CRD resource + crdName := "testtypes.testgroup.testing.cert-manager.io" + restCfg, stop := framework.RunControlPlane(t, context.Background(), + framework.WithCRDDirectory("../../../../pkg/webhook/handlers/testdata/apis/testgroup/crds"), + framework.WithWebhookConversionHandler(handlers.NewSchemeBackedConverter(testlogger.NewTestLogger(t), scheme))) + defer stop() + + // Ensure the OpenAPI endpoint has been updated with the TestType CRD + framework.WaitForOpenAPIResourcesToBeLoaded(t, ctx, restCfg, schema.GroupVersionKind{ + Group: "testgroup.testing.cert-manager.io", + Version: "v1", + Kind: "TestType", + }) + + // Create an API client + cl, err := client.New(restCfg, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + + // Fetch a copy of the recently created TestType CRD + crd := &apiext.CustomResourceDefinition{} + if err := cl.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil { + t.Fatal(err) + } + + // Identify the current storage version and one non-storage version for this CRD. + storageVersion, nonStorageVersion := versionsForCRD(crd) + if storageVersion == "" || nonStorageVersion == "" { + t.Fatal("this test requires testdata with both a storage and non-storage version set") + } + + // Ensure the original storage version is the only one on the CRD + if len(crd.Status.StoredVersions) != 1 || crd.Status.StoredVersions[0] != storageVersion { + t.Errorf("Expected status.storedVersions to only contain the storage version %q but it was: %v", storageVersion, crd.Status.StoredVersions) + } + + // Create a resource + obj := newResourceAtVersion(t, storageVersion) + if err := cl.Create(ctx, obj); err != nil { + t.Errorf("Failed to create test resource: %v", err) + } + + // We expect this to force a migration + migrator := migrateapiversion.NewMigrator(cl, true, os.Stdout, os.Stderr) + migrated, err := migrator.Run(ctx, storageVersion, []string{crdName}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !migrated { + t.Errorf("expected migrator to run due to skip flag being set") + } +} + +func versionsForCRD(crd *apiext.CustomResourceDefinition) (storage, nonstorage string) { + storageVersion := "" + nonStorageVersion := "" + for _, v := range crd.Spec.Versions { + if v.Storage { + storageVersion = v.Name + } else { + nonStorageVersion = v.Name + } + if storageVersion != "" && nonStorageVersion != "" { + break + } + } + + return storageVersion, nonStorageVersion +} + +func setStorageVersion(crd *apiext.CustomResourceDefinition, newStorageVersion string) { + for i, v := range crd.Spec.Versions { + if v.Name == newStorageVersion { + v.Storage = true + } else if v.Storage { + v.Storage = false + } + crd.Spec.Versions[i] = v + } +} + +func removeAPIVersion(crd *apiext.CustomResourceDefinition, version string) { + var newVersions []apiext.CustomResourceDefinitionVersion + for _, v := range crd.Spec.Versions { + if v.Name != version { + newVersions = append(newVersions, v) + } + } + crd.Spec.Versions = newVersions +}