weiguoqiang/cert-manager
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #4711 from munnerz/upgrade-migrate-tool

Browse Source 
Add 'cmctl upgrade migrate' tool to assist in v1.7 upgrade
This commit is contained in:
jetstack-bot 2022-01-06 17:24:22 +00:00 committed by GitHub
commit 523e0c817a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 963 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/ctl/BUILD.bazel
View File

@ -61,6 +61,7 @@ filegroup(
"//cmd/ctl/pkg/install:all-srcs",
"//cmd/ctl/pkg/renew:all-srcs",
"//cmd/ctl/pkg/status:all-srcs",
"//cmd/ctl/pkg/upgrade:all-srcs",
"//cmd/ctl/pkg/version:all-srcs",
],
tags = ["automanaged"],

1
cmd/ctl/pkg/build/commands/BUILD.bazel
View File

@ -16,6 +16,7 @@ go_library(
"//cmd/ctl/pkg/inspect:go_default_library",
"//cmd/ctl/pkg/renew:go_default_library",
"//cmd/ctl/pkg/status:go_default_library",
"//cmd/ctl/pkg/upgrade:go_default_library",
"//cmd/ctl/pkg/version:go_default_library",
"@com_github_spf13_cobra//:go_default_library",
"@io_k8s_cli_runtime//pkg/genericclioptions:go_default_library",

2
cmd/ctl/pkg/build/commands/commands.go
View File

@ -33,6 +33,7 @@ import (
"github.com/jetstack/cert-manager/cmd/ctl/pkg/inspect"
"github.com/jetstack/cert-manager/cmd/ctl/pkg/renew"
"github.com/jetstack/cert-manager/cmd/ctl/pkg/status"
"github.com/jetstack/cert-manager/cmd/ctl/pkg/upgrade"
"github.com/jetstack/cert-manager/cmd/ctl/pkg/version"
)
@ -56,6 +57,7 @@ func Commands() []RegisterCommandFunc {
approve.NewCmdApprove,
deny.NewCmdDeny,
check.NewCmdCheck,
upgrade.NewCmdUpgrade,
// Experimental features
experimental.NewCmdExperimental,

30
cmd/ctl/pkg/upgrade/BUILD.bazel Normal file
View File

@ -0,0 +1,30 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["upgrade.go"],
importpath = "github.com/jetstack/cert-manager/cmd/ctl/pkg/upgrade",
visibility = ["//visibility:public"],
deps = [
"//cmd/ctl/pkg/upgrade/migrateapiversion:go_default_library",
"@com_github_spf13_cobra//:go_default_library",
"@io_k8s_cli_runtime//pkg/genericclioptions:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//cmd/ctl/pkg/upgrade/migrateapiversion:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

46
cmd/ctl/pkg/upgrade/migrateapiversion/BUILD.bazel Normal file
View File

@ -0,0 +1,46 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"command.go",
"migrator.go",
],
importpath = "github.com/jetstack/cert-manager/cmd/ctl/pkg/upgrade/migrateapiversion",
visibility = ["//visibility:public"],
deps = [
"//cmd/ctl/pkg/build:go_default_library",
"//cmd/ctl/pkg/factory:go_default_library",
"//internal/apis/acme/install:go_default_library",
"//internal/apis/certmanager/install:go_default_library",
"@com_github_spf13_cobra//:go_default_library",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/install:go_default_library",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:go_default_library",
"@io_k8s_apimachinery//pkg/api/errors:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured:go_default_library",
"@io_k8s_apimachinery//pkg/runtime:go_default_library",
"@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
"@io_k8s_apimachinery//pkg/util/sets:go_default_library",
"@io_k8s_apimachinery//pkg/util/wait:go_default_library",
"@io_k8s_cli_runtime//pkg/genericclioptions:go_default_library",
"@io_k8s_client_go//util/retry:go_default_library",
"@io_k8s_kubectl//pkg/cmd/util:go_default_library",
"@io_k8s_kubectl//pkg/util/i18n:go_default_library",
"@io_k8s_kubectl//pkg/util/templates:go_default_library",
"@io_k8s_sigs_controller_runtime//pkg/client:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

140
cmd/ctl/pkg/upgrade/migrateapiversion/command.go Normal file
View File

@ -0,0 +1,140 @@
/*
Copyright 2022 The cert-manager Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package migrateapiversion
import (
"context"
"github.com/spf13/cobra"
apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/cli-runtime/pkg/genericclioptions"
cmdutil "k8s.io/kubectl/pkg/cmd/util"
"k8s.io/kubectl/pkg/util/i18n"
"k8s.io/kubectl/pkg/util/templates"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/jetstack/cert-manager/cmd/ctl/pkg/build"
"github.com/jetstack/cert-manager/cmd/ctl/pkg/factory"
acmeinstall "github.com/jetstack/cert-manager/internal/apis/acme/install"
cminstall "github.com/jetstack/cert-manager/internal/apis/certmanager/install"
)
var (
long = templates.LongDesc(i18n.T(`
Ensures resources in your Kubernetes cluster are persisted in the v1 API version.
This must be run prior to upgrading to ensure your cluster is ready to upgrade to cert-manager v1.7 and beyond.
This command must be run with a cluster running cert-manager v1.0 or greater.`))
example = templates.Examples(i18n.T(build.WithTemplate(`
# Check the cert-manager installation is ready to be upgraded to v1.7 and perform necessary migrations
# to ensure that the kube-apiserver has stored only v1 API versions.
{{.BuildName}} upgrade migrate-api-version
# Force migrations to be run, even if the 'status.storedVersion' field on the CRDs does not contain
# old, deprecated API versions.
# This should only be used if you have manually edited/patched the CRDs already.
# It will force a read and a write of ALL cert-manager resources unconditionally.
{{.BuildName}} upgrade migrate-api-version --skip-stored-version-check
`)))
)
// Options is a struct to support renew command
type Options struct {
genericclioptions.IOStreams
*factory.Factory
client client.Client
skipStoredVersionCheck bool
qps float32
burst int
}
// NewOptions returns initialized Options
func NewOptions(ioStreams genericclioptions.IOStreams) *Options {
return &Options{
IOStreams: ioStreams,
}
}
// NewCmdMigrate returns a cobra command for updating resources in an apiserver
// to force a new storage version to be used.
func NewCmdMigrate(ctx context.Context, ioStreams genericclioptions.IOStreams) *cobra.Command {
o := NewOptions(ioStreams)
cmd := &cobra.Command{
Use: "migrate-api-version",
Short: "Migrate all existing persisted cert-manager resources to the v1 API version",
Long: long,
Example: example,
Run: func(cmd *cobra.Command, args []string) {
cmdutil.CheckErr(o.Validate(args))
cmdutil.CheckErr(o.Complete())
cmdutil.CheckErr(o.Run(ctx, args))
},
}
cmd.Flags().BoolVar(&o.skipStoredVersionCheck, "skip-stored-version-check", o.skipStoredVersionCheck, ""+
"If true, all resources will be read and written regardless of the 'status.storedVersions' on the CRD resource. "+
"Use this mode if you have previously manually modified the 'status.storedVersions' field on CRD resources.")
cmd.Flags().Float32Var(&o.qps, "qps", 5, "Indicates the maximum QPS to the apiserver from the client.")
cmd.Flags().IntVar(&o.burst, "burst", 10, "Maximum burst value for queries set to the apiserver from the client.")
o.Factory = factory.New(ctx, cmd)
return cmd
}
// Validate validates the provided options
func (o *Options) Validate(_ []string) error {
return nil
}
// Complete takes the command arguments and factory and infers any remaining options.
func (o *Options) Complete() error {
var err error
scheme := runtime.NewScheme()
apiextinstall.Install(scheme)
cminstall.Install(scheme)
acmeinstall.Install(scheme)
if o.qps != 0 {
o.RESTConfig.QPS = o.qps
}
if o.burst != 0 {
o.RESTConfig.Burst = o.burst
}
o.client, err = client.New(o.RESTConfig, client.Options{Scheme: scheme})
if err != nil {
return err
}
return nil
}
// Run executes renew command
func (o *Options) Run(ctx context.Context, args []string) error {
_, err := NewMigrator(o.client, o.skipStoredVersionCheck, o.Out, o.ErrOut).Run(ctx, "v1", []string{
"certificates.cert-manager.io",
"certificaterequests.cert-manager.io",
"issuers.cert-manager.io",
"clusterissuers.cert-manager.io",
"orders.acme.cert-manager.io",
"challenges.acme.cert-manager.io",
})
return err
}

281
cmd/ctl/pkg/upgrade/migrateapiversion/migrator.go Normal file
View File

@ -0,0 +1,281 @@
/*
Copyright 2022 The cert-manager Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package migrateapiversion
import (
"context"
"fmt"
"io"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
type Migrator struct {
// Client used for API interactions
Client client.Client
// If true, skip checking the 'status.storedVersion' before running the migration.
// By default, migration will only be run if the CRD contains storedVersions other
// than the desired target version.
SkipStoredVersionCheck bool
// Writers to write informational & error messages to
Out, ErrOut io.Writer
}
// NewMigrator creates a new migrator with the given API client.
// If either of out or errOut are nil, log messages will be discarded.
func NewMigrator(client client.Client, skipStoredVersionCheck bool, out, errOut io.Writer) *Migrator {
if out == nil {
out = io.Discard
}
if errOut == nil {
errOut = io.Discard
}
return &Migrator{
Client: client,
SkipStoredVersionCheck: skipStoredVersionCheck,
Out: out,
ErrOut: errOut,
}
}
// Run begins the migration of all the named CRDs.
// It will attempt to migrate all resources defined as part of these CRDs to the
// given 'targetVersion', and after completion will update the `status.storedVersions`
// field on the corresponding CRD version to only contain the given targetVersion.
// Returns 'true' if a migration was actually performed, and false if migration was not required.
func (m *Migrator) Run(ctx context.Context, targetVersion string, names []string) (bool, error) {
fmt.Fprintf(m.Out, "Checking all CustomResourceDefinitions have storage version set to '%s'\n", targetVersion)
allTargetVersion, allCRDs, err := m.ensureCRDStorageVersionEquals(ctx, targetVersion, names)
if err != nil {
return false, err
}
if !allTargetVersion {
fmt.Fprintf(m.ErrOut, "It looks like you are running a version of cert-manager that does not set the storage version of CRDs to %q. You MUST upgrade to cert-manager v1.0-v1.6 before migrating resources for v1.7.\n", targetVersion)
return false, fmt.Errorf("preflight checks failed")
}
fmt.Fprintf(m.Out, "All CustomResourceDefinitions have %q configured as the storage version.\n", targetVersion)
crdsRequiringMigration := allCRDs
if !m.SkipStoredVersionCheck {
fmt.Fprintf(m.Out, "Looking for CRDs that contain resources that require migrating to %q...\n", targetVersion)
crdsRequiringMigration, err = m.discoverCRDsRequiringMigration(ctx, targetVersion, names)
if err != nil {
fmt.Fprintf(m.ErrOut, "Failed to determine resource types that require migration: %v\n", err)
return false, err
}
if len(crdsRequiringMigration) == 0 {
fmt.Fprintln(m.Out, "Nothing to do. cert-manager CRDs do not have 'status.storedVersions' containing old API versions. You may proceed to upgrade to cert-manager v1.7.")
return false, nil
}
} else {
fmt.Fprintln(m.Out, "Forcing migration of all CRD resources as --skip-stored-version-check=true")
}
fmt.Fprintf(m.Out, "Found %d resource types that require migration:\n", len(crdsRequiringMigration))
for _, crd := range crdsRequiringMigration {
fmt.Fprintf(m.Out, " - %s\n", crd.Name)
}
for _, crd := range crdsRequiringMigration {
if err := m.migrateResourcesForCRD(ctx, crd); err != nil {
fmt.Fprintf(m.ErrOut, "Failed to migrate resource: %v\n", err)
return false, err
}
}
fmt.Fprintf(m.Out, "Patching CRD resources to set 'status.storedVersions' to %q...\n", targetVersion)
if err := m.patchCRDStoredVersions(ctx, crdsRequiringMigration); err != nil {
fmt.Fprintf(m.ErrOut, "Failed to patch 'status.storedVersions' field: %v\n", err)
return false, err
}
fmt.Fprintln(m.Out, "Successfully migrated all cert-manager resource types. It is now safe to proceed with upgrading to cert-manager v1.7.")
return true, nil
}
func (m *Migrator) ensureCRDStorageVersionEquals(ctx context.Context, vers string, names []string) (bool, []*apiext.CustomResourceDefinition, error) {
var crds []*apiext.CustomResourceDefinition
for _, crdName := range names {
crd := &apiext.CustomResourceDefinition{}
if err := m.Client.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
return false, nil, err
}
// Discover the storage version
storageVersion := storageVersionForCRD(crd)
if storageVersion != vers {
fmt.Fprintf(m.Out, "CustomResourceDefinition object %q has storage version set to %q.\n", crdName, storageVersion)
return false, nil, nil
}
crds = append(crds, crd)
}
return true, crds, nil
}
func (m *Migrator) discoverCRDsRequiringMigration(ctx context.Context, desiredStorageVersion string, names []string) ([]*apiext.CustomResourceDefinition, error) {
var requireMigration []*apiext.CustomResourceDefinition
for _, name := range names {
crd := &apiext.CustomResourceDefinition{}
if err := m.Client.Get(ctx, client.ObjectKey{Name: name}, crd); err != nil {
return nil, err
}
// If no versions are stored, there's nothing to migrate.
if len(crd.Status.StoredVersions) == 0 {
continue
}
// If more than one entry exists in `storedVersions` OR if the only element in there is not
// the desired version, perform a migration.
if len(crd.Status.StoredVersions) > 1 || crd.Status.StoredVersions[0] != desiredStorageVersion {
requireMigration = append(requireMigration, crd)
}
}
return requireMigration, nil
}
func (m *Migrator) migrateResourcesForCRD(ctx context.Context, crd *apiext.CustomResourceDefinition) error {
startTime := time.Now()
fmt.Fprintf(m.Out, "Migrating %q objects in group %q - this may take a while (started at %s)...\n", crd.Spec.Names.Kind, crd.Spec.Group, startTime.Format(time.Stamp))
list := &unstructured.UnstructuredList{}
list.SetGroupVersionKind(schema.GroupVersionKind{
Group: crd.Spec.Group,
Version: storageVersionForCRD(crd),
Kind: crd.Spec.Names.ListKind,
})
if err := m.Client.List(ctx, list); err != nil {
return err
}
fmt.Fprintf(m.Out, " %d resources to migrate\n", len(list.Items))
for _, obj := range list.Items {
// retry on any kind of error to handle cases where e.g. the network connection to the apiserver fails
if err := retry.OnError(wait.Backoff{
Duration: time.Second, // wait 1s between attempts
Steps: 3, // allow up to 3 attempts per object
}, func(err error) bool {
// Retry on any errors that are not otherwise skipped/ignored
return handleUpdateErr(err) != nil
}, func() error { return m.Client.Update(ctx, &obj) }); handleUpdateErr(err) != nil {
return err
}
}
fmt.Fprintf(m.Out, " Successfully migrated %d %s objects in %s\n", len(list.Items), crd.Spec.Names.Kind, time.Now().Sub(startTime).Round(time.Second))
return nil
}
// patchCRDStoredVersions will patch the `status.storedVersions` field of all passed in CRDs to be
// set to an array containing JUST the current storage version.
// This is only safe to run after a successful migration (i.e. a read/write of all resources of the given CRD type).
func (m *Migrator) patchCRDStoredVersions(ctx context.Context, crds []*apiext.CustomResourceDefinition) error {
for _, crd := range crds {
// fetch a fresh copy of the CRD to avoid any conflict errors
freshCRD := &apiext.CustomResourceDefinition{}
if err := m.Client.Get(ctx, client.ObjectKey{Name: crd.Name}, freshCRD); err != nil {
return err
}
// Check the latest copy of the CRD to ensure that:
// 1) the storage version is the same as it was at the start of the migration
// 2) the status.storedVersion field has not changed, and if it has, it has only added the new/desired storage version
// This helps to avoid cases where the storage version was changed by a third-party midway through the migration,
// which could lead to corrupted apiservers when we patch the status.storedVersions field below.
expectedStorageVersion := storageVersionForCRD(crd)
if storageVersionForCRD(freshCRD) != expectedStorageVersion {
return newUnexpectedChangeError(crd)
}
newlyAddedVersions := storedVersionsAdded(crd, freshCRD)
if newlyAddedVersions.Len() != 0 && !newlyAddedVersions.Equal(sets.NewString(expectedStorageVersion)) {
return newUnexpectedChangeError(crd)
}
// Set the `status.storedVersions` field to the target storage version
freshCRD.Status.StoredVersions = []string{storageVersionForCRD(crd)}
if err := m.Client.Status().Update(ctx, freshCRD); err != nil {
return err
}
}
return nil
}
// storageVersionForCRD discovers the storage version for a given CRD.
func storageVersionForCRD(crd *apiext.CustomResourceDefinition) string {
storageVersion := ""
for _, v := range crd.Spec.Versions {
if v.Storage {
storageVersion = v.Name
break
}
}
return storageVersion
}
// storedVersionsAdded returns a list of any versions added to the `status.storedVersions` field on
// a CRD resource.
func storedVersionsAdded(old, new *apiext.CustomResourceDefinition) sets.String {
oldStoredVersions := sets.NewString(old.Status.StoredVersions...)
newStoredVersions := sets.NewString(new.Status.StoredVersions...)
return newStoredVersions.Difference(oldStoredVersions)
}
// newUnexpectedChangeError creates a new 'error' that informs users that a change to the CRDs
// was detected during the migration process and so the migration must be re-run.
func newUnexpectedChangeError(crd *apiext.CustomResourceDefinition) error {
errorFmt := "" +
"The CRD %q unexpectedly changed during the migration. " +
"This means that either an object was persisted in a non-storage version during the migration, " +
"or the storage version was changed by someone else (or some automated deployment tooling) whilst the migration " +
"was in progress.\n\n" +
"All automated deployment tooling should be in a 'stable state' (i.e. no upgrades to cert-manager CRDs should be" +
"in progress whilst the migration is running).\n\n" +
"Please ensure no changes to the CRDs are made during the migration process and re-run the migration until you" +
"no longer see this message."
return fmt.Errorf(errorFmt, crd.Name)
}
// handleUpdateErr will absorb certain types of errors that we know can be skipped/passed on
// during a migration of a particular object.
func handleUpdateErr(err error) error {
if err == nil {
return nil
}
// If the resource no longer exists, don't return the error as the object no longer
// needs updating to the new API version.
if apierrors.IsNotFound(err) {
return nil
}
// If there was a conflict, another client must have written the object already which
// means we don't need to force an update.
if apierrors.IsConflict(err) {
return nil
}
return err
}

38
cmd/ctl/pkg/upgrade/upgrade.go Normal file
View File

@ -0,0 +1,38 @@
/*
Copyright 2022 The cert-manager Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"context"
"github.com/spf13/cobra"
"k8s.io/cli-runtime/pkg/genericclioptions"
"github.com/jetstack/cert-manager/cmd/ctl/pkg/upgrade/migrateapiversion"
)
func NewCmdUpgrade(ctx context.Context, ioStreams genericclioptions.IOStreams) *cobra.Command {
cmds := &cobra.Command{
Use: "upgrade",
Short: "Tools that assist in upgrading cert-manager",
Long: `Note: this command does NOT actually upgrade cert-manager installations`,
}
cmds.AddCommand(migrateapiversion.NewCmdMigrate(ctx, ioStreams))
return cmds
}

1
test/integration/ctl/BUILD.bazel
View File

@ -49,6 +49,7 @@ filegroup(
srcs = [
":package-srcs",
"//test/integration/ctl/install_framework:all-srcs",
"//test/integration/ctl/migrate:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],

39
test/integration/ctl/migrate/BUILD.bazel Normal file
View File

@ -0,0 +1,39 @@
load("@io_bazel_rules_go//go:def.bzl", "go_test")
go_test(
name = "go_default_test",
srcs = ["ctl_upgrade_migrate_test.go"],
data = [
"//pkg/webhook/handlers/testdata/apis/testgroup/crds:all-srcs",
],
deps = [
"//cmd/ctl/pkg/upgrade/migrateapiversion:go_default_library",
"//pkg/webhook/handlers:go_default_library",
"//pkg/webhook/handlers/testdata/apis/testgroup/install:go_default_library",
"//pkg/webhook/handlers/testdata/apis/testgroup/v1:go_default_library",
"//pkg/webhook/handlers/testdata/apis/testgroup/v2:go_default_library",
"//test/integration/framework:go_default_library",
"@com_github_go_logr_logr//testing:go_default_library",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/install:go_default_library",
"@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured:go_default_library",
"@io_k8s_apimachinery//pkg/runtime:go_default_library",
"@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
"@io_k8s_sigs_controller_runtime//pkg/client:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

384
test/integration/ctl/migrate/ctl_upgrade_migrate_test.go Normal file
View File

@ -0,0 +1,384 @@
/*
Copyright 2022 The cert-manager Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package migrate
import (
"context"
"os"
"testing"
"time"
testlogger "github.com/go-logr/logr/testing"
apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"
apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/jetstack/cert-manager/cmd/ctl/pkg/upgrade/migrateapiversion"
"github.com/jetstack/cert-manager/pkg/webhook/handlers"
"github.com/jetstack/cert-manager/pkg/webhook/handlers/testdata/apis/testgroup/install"
"github.com/jetstack/cert-manager/pkg/webhook/handlers/testdata/apis/testgroup/v1"
"github.com/jetstack/cert-manager/pkg/webhook/handlers/testdata/apis/testgroup/v2"
"github.com/jetstack/cert-manager/test/integration/framework"
)
// Create a test resource at a given version.
func newResourceAtVersion(t *testing.T, version string) client.Object {
switch version {
case "v1":
return &v1.TestType{
ObjectMeta: metav1.ObjectMeta{
Name: "object",
Namespace: "default",
},
TestField: "abc",
TestFieldImmutable: "def",
}
case "v2":
return &v2.TestType{
ObjectMeta: metav1.ObjectMeta{
Name: "object",
Namespace: "default",
},
TestField: "abc",
TestFieldImmutable: "def",
}
default:
t.Fatalf("unknown version %q", version)
}
return nil
}
func newScheme() *runtime.Scheme {
scheme := runtime.NewScheme()
apiextinstall.Install(scheme)
install.Install(scheme)
return scheme
}
func TestCtlUpgradeMigrate(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
// Create the control plane with the TestType conversion handlers registered
scheme := newScheme()
// name of the testtype CRD resource
crdName := "testtypes.testgroup.testing.cert-manager.io"
restCfg, stop := framework.RunControlPlane(t, context.Background(),
framework.WithCRDDirectory("../../../../pkg/webhook/handlers/testdata/apis/testgroup/crds"),
framework.WithWebhookConversionHandler(handlers.NewSchemeBackedConverter(testlogger.NewTestLogger(t), scheme)))
defer stop()
// Ensure the OpenAPI endpoint has been updated with the TestType CRD
framework.WaitForOpenAPIResourcesToBeLoaded(t, ctx, restCfg, schema.GroupVersionKind{
Group: "testgroup.testing.cert-manager.io",
Version: "v1",
Kind: "TestType",
})
// Create an API client
cl, err := client.New(restCfg, client.Options{Scheme: scheme})
if err != nil {
t.Fatal(err)
}
// Fetch a copy of the recently created TestType CRD
crd := &apiext.CustomResourceDefinition{}
if err := cl.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
t.Fatal(err)
}
// Identify the current storage version and one non-storage version for this CRD.
// We'll be creating objects and then changing the storage version on the CRD to
// the 'nonStorageVersion' and ensuring the migration/upgrade is successful.
storageVersion, nonStorageVersion := versionsForCRD(crd)
if storageVersion == "" || nonStorageVersion == "" {
t.Fatal("this test requires testdata with both a storage and non-storage version set")
}
// Ensure the original storage version is the only one on the CRD
if len(crd.Status.StoredVersions) != 1 || crd.Status.StoredVersions[0] != storageVersion {
t.Errorf("Expected status.storedVersions to only contain the storage version %q but it was: %v", storageVersion, crd.Status.StoredVersions)
}
// Create a resource
obj := newResourceAtVersion(t, storageVersion)
if err := cl.Create(ctx, obj); err != nil {
t.Errorf("Failed to create test resource: %v", err)
}
// Set the storage version to the 'nonStorageVersion'
setStorageVersion(crd, nonStorageVersion)
if err := cl.Update(ctx, crd); err != nil {
t.Fatalf("Failed to update CRD storage version: %v", err)
}
if len(crd.Status.StoredVersions) != 2 || crd.Status.StoredVersions[0] != storageVersion || crd.Status.StoredVersions[1] != nonStorageVersion {
t.Fatalf("Expected status.storedVersions to contain [%s, %s] but it was: %v", storageVersion, nonStorageVersion, crd.Status.StoredVersions)
}
// Run the migrator and migrate all objects to the 'nonStorageVersion' (which is now the new storage version)
migrator := migrateapiversion.NewMigrator(cl, false, os.Stdout, os.Stderr)
migrated, err := migrator.Run(ctx, nonStorageVersion, []string{crdName})
if err != nil {
t.Errorf("migrator failed to run: %v", err)
}
if !migrated {
t.Errorf("migrator didn't actually perform a migration")
}
// Check the status.storedVersions field to ensure it only contains one element
crd = &apiext.CustomResourceDefinition{}
if err := cl.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
t.Fatal(err)
}
if len(crd.Status.StoredVersions) != 1 || crd.Status.StoredVersions[0] != nonStorageVersion {
t.Fatalf("Expected status.storedVersions to be %q but it was: %v", nonStorageVersion, crd.Status.StoredVersions)
}
// Remove the previous storage version from the CRD and update it
removeAPIVersion(crd, storageVersion)
if err := cl.Update(ctx, crd); err != nil {
t.Fatalf("Failed to remove old API version: %v", err)
}
// Attempt to read a resource list in the new API version
objList := &unstructured.UnstructuredList{}
objList.SetGroupVersionKind(schema.GroupVersionKind{
Group: crd.Spec.Group,
Version: nonStorageVersion,
Kind: crd.Spec.Names.ListKind,
})
if err := cl.List(ctx, objList); err != nil {
t.Fatalf("Failed to list objects (gvk %v): %v", objList.GroupVersionKind(), err)
}
if len(objList.Items) != 1 {
t.Fatalf("Expected a single TestType resource to exist")
}
}
func TestCtlUpgradeMigrate_FailsIfStorageVersionDoesNotEqualTargetVersion(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
// Create the control plane with the TestType conversion handlers registered
scheme := newScheme()
// name of the testtype CRD resource
crdName := "testtypes.testgroup.testing.cert-manager.io"
restCfg, stop := framework.RunControlPlane(t, context.Background(),
framework.WithCRDDirectory("../../../../pkg/webhook/handlers/testdata/apis/testgroup/crds"),
framework.WithWebhookConversionHandler(handlers.NewSchemeBackedConverter(testlogger.NewTestLogger(t), scheme)))
defer stop()
// Ensure the OpenAPI endpoint has been updated with the TestType CRD
framework.WaitForOpenAPIResourcesToBeLoaded(t, ctx, restCfg, schema.GroupVersionKind{
Group: "testgroup.testing.cert-manager.io",
Version: "v1",
Kind: "TestType",
})
// Create an API client
cl, err := client.New(restCfg, client.Options{Scheme: scheme})
if err != nil {
t.Fatal(err)
}
// Fetch a copy of the recently created TestType CRD
crd := &apiext.CustomResourceDefinition{}
if err := cl.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
t.Fatal(err)
}
// Identify the current storage version and one non-storage version for this CRD.
storageVersion, nonStorageVersion := versionsForCRD(crd)
if storageVersion == "" || nonStorageVersion == "" {
t.Fatal("this test requires testdata with both a storage and non-storage version set")
}
// We expect this to fail, as we are attempting to migrate to the 'nonStorageVersion'.
migrator := migrateapiversion.NewMigrator(cl, false, os.Stdout, os.Stderr)
migrated, err := migrator.Run(ctx, nonStorageVersion, []string{crdName})
if err == nil {
t.Errorf("expected an error to be returned but we got none")
}
if err.Error() != "preflight checks failed" {
t.Errorf("unexpected error: %v", err)
}
if migrated {
t.Errorf("migrator ran but it should not have")
}
}
func TestCtlUpgradeMigrate_SkipsMigrationIfNothingToDo(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
// Create the control plane with the TestType conversion handlers registered
scheme := newScheme()
// name of the testtype CRD resource
crdName := "testtypes.testgroup.testing.cert-manager.io"
restCfg, stop := framework.RunControlPlane(t, context.Background(),
framework.WithCRDDirectory("../../../../pkg/webhook/handlers/testdata/apis/testgroup/crds"),
framework.WithWebhookConversionHandler(handlers.NewSchemeBackedConverter(testlogger.NewTestLogger(t), scheme)))
defer stop()
// Ensure the OpenAPI endpoint has been updated with the TestType CRD
framework.WaitForOpenAPIResourcesToBeLoaded(t, ctx, restCfg, schema.GroupVersionKind{
Group: "testgroup.testing.cert-manager.io",
Version: "v1",
Kind: "TestType",
})
// Create an API client
cl, err := client.New(restCfg, client.Options{Scheme: scheme})
if err != nil {
t.Fatal(err)
}
// Fetch a copy of the recently created TestType CRD
crd := &apiext.CustomResourceDefinition{}
if err := cl.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
t.Fatal(err)
}
// Identify the current storage version and one non-storage version for this CRD.
storageVersion, nonStorageVersion := versionsForCRD(crd)
if storageVersion == "" || nonStorageVersion == "" {
t.Fatal("this test requires testdata with both a storage and non-storage version set")
}
// Ensure the original storage version is the only one on the CRD
if len(crd.Status.StoredVersions) != 1 || crd.Status.StoredVersions[0] != storageVersion {
t.Errorf("Expected status.storedVersions to only contain the storage version %q but it was: %v", storageVersion, crd.Status.StoredVersions)
}
// Create a resource
obj := newResourceAtVersion(t, storageVersion)
if err := cl.Create(ctx, obj); err != nil {
t.Errorf("Failed to create test resource: %v", err)
}
// We expect this to succeed and for the migration to not be run
migrator := migrateapiversion.NewMigrator(cl, false, os.Stdout, os.Stderr)
migrated, err := migrator.Run(ctx, storageVersion, []string{crdName})
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if migrated {
t.Errorf("migrator ran but it should not have")
}
}
func TestCtlUpgradeMigrate_ForcesMigrationIfSkipStoredVersionCheckIsEnabled(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
// Create the control plane with the TestType conversion handlers registered
scheme := newScheme()
// name of the testtype CRD resource
crdName := "testtypes.testgroup.testing.cert-manager.io"
restCfg, stop := framework.RunControlPlane(t, context.Background(),
framework.WithCRDDirectory("../../../../pkg/webhook/handlers/testdata/apis/testgroup/crds"),
framework.WithWebhookConversionHandler(handlers.NewSchemeBackedConverter(testlogger.NewTestLogger(t), scheme)))
defer stop()
// Ensure the OpenAPI endpoint has been updated with the TestType CRD
framework.WaitForOpenAPIResourcesToBeLoaded(t, ctx, restCfg, schema.GroupVersionKind{
Group: "testgroup.testing.cert-manager.io",
Version: "v1",
Kind: "TestType",
})
// Create an API client
cl, err := client.New(restCfg, client.Options{Scheme: scheme})
if err != nil {
t.Fatal(err)
}
// Fetch a copy of the recently created TestType CRD
crd := &apiext.CustomResourceDefinition{}
if err := cl.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
t.Fatal(err)
}
// Identify the current storage version and one non-storage version for this CRD.
storageVersion, nonStorageVersion := versionsForCRD(crd)
if storageVersion == "" || nonStorageVersion == "" {
t.Fatal("this test requires testdata with both a storage and non-storage version set")
}
// Ensure the original storage version is the only one on the CRD
if len(crd.Status.StoredVersions) != 1 || crd.Status.StoredVersions[0] != storageVersion {
t.Errorf("Expected status.storedVersions to only contain the storage version %q but it was: %v", storageVersion, crd.Status.StoredVersions)
}
// Create a resource
obj := newResourceAtVersion(t, storageVersion)
if err := cl.Create(ctx, obj); err != nil {
t.Errorf("Failed to create test resource: %v", err)
}
// We expect this to force a migration
migrator := migrateapiversion.NewMigrator(cl, true, os.Stdout, os.Stderr)
migrated, err := migrator.Run(ctx, storageVersion, []string{crdName})
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if !migrated {
t.Errorf("expected migrator to run due to skip flag being set")
}
}
func versionsForCRD(crd *apiext.CustomResourceDefinition) (storage, nonstorage string) {
storageVersion := ""
nonStorageVersion := ""
for _, v := range crd.Spec.Versions {
if v.Storage {
storageVersion = v.Name
} else {
nonStorageVersion = v.Name
}
if storageVersion != "" && nonStorageVersion != "" {
break
}
}
return storageVersion, nonStorageVersion
}
func setStorageVersion(crd *apiext.CustomResourceDefinition, newStorageVersion string) {
for i, v := range crd.Spec.Versions {
if v.Name == newStorageVersion {
v.Storage = true
} else if v.Storage {
v.Storage = false
}
crd.Spec.Versions[i] = v
}
}
func removeAPIVersion(crd *apiext.CustomResourceDefinition, version string) {
var newVersions []apiext.CustomResourceDefinitionVersion
for _, v := range crd.Spec.Versions {
if v.Name != version {
newVersions = append(newVersions, v)
}
}
crd.Spec.Versions = newVersions
}