From d8e7686d287e0e0390f6c175b3c91381421240c4 Mon Sep 17 00:00:00 2001 From: Haowei Cai Date: Thu, 22 Apr 2021 15:55:58 -0700 Subject: [PATCH] add a release script --- scripts/apply-hotfixes.sh | 14 +--- scripts/release.sh | 134 +++++++++++++++++++++++++++++++-- scripts/util/changelog.sh | 49 +++++++++++- scripts/util/kube_changelog.sh | 57 ++++++++++++++ 4 files changed, 232 insertions(+), 22 deletions(-) create mode 100755 scripts/util/kube_changelog.sh diff --git a/scripts/apply-hotfixes.sh b/scripts/apply-hotfixes.sh index 2013315a3..25a2b8447 100755 --- a/scripts/apply-hotfixes.sh +++ b/scripts/apply-hotfixes.sh @@ -24,21 +24,11 @@ then exit 1 fi -# Check if the current branch is a release branch (release-*) -# If it is not a release branch, don't let the patch be applied -GIT_BRANCH=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') -if ! [[ $GIT_BRANCH =~ .*release-.* ]]; then - echo Current branch: $GIT_BRANCH - echo You are not in a release branch, e.g., release-11.0, release-10.0 - echo Please switch to a release branch to run this script. - exit 1 -fi - # Patching commit for custom client behavior # UPDATE: The commit being cherry-picked is updated since the the client generated in 1adaaecd0879d7315f48259ad8d6cbd66b835385 # differs from the initial hotfix # Ref: https://github.com/kubernetes-client/python/pull/995/commits/9959273625b999ae9a8f0679c4def2ee7d699ede -git cherry-pick -n 9959273625b999ae9a8f0679c4def2ee7d699ede +git cherry-pick -n 90aa6f6ab9a391e35d63a84af27e14cc5d5ce947 if [ $? -eq 0 ] then echo Succesfully patched changes for custom client behavior @@ -77,7 +67,7 @@ fi; # Patching commits for Tolerating Null Sources on Projected Volumes # TODO: remove this patch when we release v20 clients # Ref: https://github.com/kubernetes-client/python/pull/1497 -git cherry-pick -n f3dbc8cbf1ab2aaf5e3bd8c0f0fc068e67823971 +git cherry-pick -n ee0e332776d9002bea07d328d49e90ed8c221795 if [ $? -eq 0 ] then echo Succesfully patched changes for Tolerating Null Sources on Projected Volumes diff --git a/scripts/release.sh b/scripts/release.sh index 999c2b23d..bf576c6c3 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -1,10 +1,128 @@ -# This file is intended to document release steps. -# Verify each step's result before calling the next command. -# It is documented here with the intention of being automated -# as a shell script later. +#!/bin/bash -echo 'git clean -xdf' -echo 'python setup.py sdist' -echo 'python setup.py bdist_wheel --universal' -echo 'twine upload dist/* -r https://upload.pypi.org/legacy/ -u kubernetes' +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Workflow +# 1. [master branch] update existing snapshot (include API change for a new alpha/beta/GA +# release) +# - add a new snapshot or reuse the existing snapshot, the latter means either +# API change happened in a k8s patch release, or we want to include some new +# python / python-base change in the release note +# - API change w/ release notes +# - master change w/ release notes +# - submodule change w/ release notes +# 2. [master branch] create new snapshot (include API change for a new alpha release) +# - add a new snapshot or reuse the existing snapshot, the latter means either +# API change happened in a k8s patch release, or we want to include some new +# python / python-base change in the release note +# - API change w/ release notes +# - master change w/ release notes +# - submodule change w/ release notes +# 3. [release branch] create a new release +# - pull master +# - it's possible that master has new changes after the latest snaphost, +# update CHANGELOG accordingly +# - for generated file, resolve conflict by committing the master version +# - abort if a snapshot doesn't exist +# - generate client change, abort if API change is detected +# - CHANGELOG: latest snapshot becomes the release, create a new snapshot +# section that reflect the master branch state +# - README: add the release to README +# - an extra PR to update CHANGELOG and README in master in sync with this new +# release +# +# Difference between 1&2: API change release notes +# +# TODO(roycaihw): +# - add user input validation +# - add function input validaiton (release/version strings start with 'v' or not) +# - automatically send a PR; provide useful links for review +# - master branch diff: https://github.com/kubernetes-client/python/compare/commit1..commit2 +# - python base diff: https://github.com/kubernetes-client/python-base/compare/commit1..commit2 +# - Kubernetes changelog, e.g. https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md +# - add debug log +# - add a sentence about "changes since {last release}". In most cases our +# releases should be sequential. This script (the workflow above) is based on +# this assumption, and we should make the release note clear about that. +# +# Usage: +# $ KUBERNETES_BRANCH=release-1.19 CLIENT_VERSION=19.0.0-snapshot DEVELOPMENT_STATUS="3 - Alpha" scripts/release.sh + +set -o errexit +set -o nounset +set -o pipefail + +repo_root="$(git rev-parse --show-toplevel)" +declare -r repo_root +cd "${repo_root}" + +source scripts/util/changelog.sh +source scripts/util/kube_changelog.sh + +old_client_version=$(python3 "scripts/constants.py" CLIENT_VERSION) +old_k8s_api_version=$(util::changelog::get_k8s_api_version "v$old_client_version") +KUBERNETES_BRANCH=${KUBERNETES_BRANCH:-$(python3 "scripts/constants.py" KUBERNETES_BRANCH)} +CLIENT_VERSION=${CLIENT_VERSION:-$(python3 "scripts/constants.py" CLIENT_VERSION)} +DEVELOPMENT_STATUS=${DEVELOPMENT_STATUS:-$(python3 "scripts/constants.py" DEVELOPMENT_STATUS)} + +# get Kubernetes API Version +new_k8s_api_version=$(util::kube_changelog::find_latest_patch_version $KUBERNETES_BRANCH) +echo "Old Kubernetes API Version: $old_k8s_api_version" +echo "New Kubernetes API Version: $new_k8s_api_version" + +sed -i "s/^KUBERNETES_BRANCH =.*$/KUBERNETES_BRANCH = \"$KUBERNETES_BRANCH\"/g" scripts/constants.py +sed -i "s/^CLIENT_VERSION =.*$/CLIENT_VERSION = \"$CLIENT_VERSION\"/g" scripts/constants.py +sed -i "s/^DEVELOPMENT_STATUS =.*$/DEVELOPMENT_STATUS = \"$DEVELOPMENT_STATUS\"/g" scripts/constants.py +git commit -am "update version constants for $CLIENT_VERSION release" + +util::changelog::update_release_api_version $CLIENT_VERSION $old_client_version $new_k8s_api_version + +# get API change release notes since $old_k8s_api_version. +# NOTE: $old_k8s_api_version may be one-minor-version behind $KUBERNETES_BRANCH, e.g. +# KUBERNETES_BRANCH=release-1.19 +# old_k8s_api_version=1.18.17 +# when we bump the minor version for the snapshot in the master branch. We +# don't need to collect release notes in release-1.18, because any API +# change in 1.18.x (x > 17) must be a cherrypick that is already included in +# release-1.19. +# TODO(roycaihw): not all Kubernetes API changes modify the OpenAPI spec. +# Download the patch and skip if the spec is not modified. Also we want to +# look at other k/k sections like "deprecation" +release_notes=$(util::kube_changelog::get_api_changelog "$KUBERNETES_BRANCH" "$old_k8s_api_version") +if [[ -n "$release_notes" ]]; then + util::changelog::write_changelog v$CLIENT_VERSION "### API Change" "$release_notes" +fi + +git commit -am "update changelog" + +# run client generator +scripts/update-client.sh + +rm -r kubernetes/test/ +git add . +git commit -m "temporary generated commit" +scripts/apply-hotfixes.sh +git reset HEAD~2 +# custom object API is hosted in gen repo. Commit API change separately for +# easier review +if [[ -z "$(git diff kubernetes/client/api/custom_objects_api.py)" ]]; then + git add kubernetes/client/api/custom_objects_api.py + git commit -m "generated client change for custom_objects" +fi +git add kubernetes/docs kubernetes/client/api/ kubernetes/client/models/ kubernetes/swagger.json.unprocessed scripts/swagger.json +git commit -m "generated API change" +git add . +git commit -m "generated client change" +echo "Release finished successfully." diff --git a/scripts/util/changelog.sh b/scripts/util/changelog.sh index 672b82336..3bc8af999 100755 --- a/scripts/util/changelog.sh +++ b/scripts/util/changelog.sh @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. + +# Utilities for parsing/writing the Python client's changelog. + changelog="$(git rev-parse --show-toplevel)/CHANGELOG.md" function util::changelog::has_release { @@ -44,7 +47,7 @@ function util::changelog::find_release_end { echo $((${releases[${next_release_index}]}-1)) } -# has_section returns if the given section exists between start and end +# has_section_in_range returns if the given section exists between start and end function util::changelog::has_section_in_range { local section="$1" local start=$2 @@ -59,7 +62,7 @@ function util::changelog::has_section_in_range { return 1 } -# find_section returns the number of the first line of the given section +# find_section_in_range returns the number of the first line of the given section function util::changelog::find_section_in_range { local section="$1" local start=$2 @@ -107,3 +110,45 @@ function util::changelog::write_changelog { # update changelog sed -i "${line_to_edit}i${release_notes}" $changelog } + +# get_api_version returns the Kubernetes API Version for the given client +# version in the changelog. +function util::changelog::get_k8s_api_version { + local client_version="$1" + + local api_version_section="Kubernetes API Version: " + # by default, find the first API version in the first 100 lines if the given + # client version isn't found + local start=0 + local end=100 + if util::changelog::has_release "$client_version"; then + start=$(util::changelog::find_release_start "$client_version") + end=$(util::changelog::find_release_end "$client_version") + fi + if ! util::changelog::has_section_in_range "$api_version_section" "$start" "$end"; then + echo "error: api version for release $client_version not found" + exit 1 + fi + + local api_version_line=$(util::changelog::find_section_in_range "$api_version_section" "$start" "$end") + echo $(sed -n ${api_version_line}p $changelog | sed "s/$api_version_section//g") +} + +function util::changelog::update_release_api_version { + local release="$1" + local old_release="$2" + local k8s_api_version="$3" + + echo "New release: $release" + echo "Old release: $old_release" + + if ! util::changelog::has_release v$old_release; then + sed -i "1i# v$release\n\nKubernetes API Version: $k8s_api_version\n\n" $changelog + return 0 + fi + start=$(util::changelog::find_release_start v$old_release) + sed -i "${start}s/# v$old_release/# v$release/" $changelog + echo "$start" + echo "$((${start}+2))" + sed -i "$((${start}+2))s/^Kubernetes API Version: .*$/Kubernetes API Version: $k8s_api_version/" $changelog +} diff --git a/scripts/util/kube_changelog.sh b/scripts/util/kube_changelog.sh new file mode 100755 index 000000000..745e70aa5 --- /dev/null +++ b/scripts/util/kube_changelog.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Utilities for parsing Kubernetes changelog. + +# find_latest_patch_version finds the latest released patch version for the +# given branch. +# We use the version to track what API surface the generated Python client +# cooresponds to, and collect all API change release notes up to that version. +# There is one tricky point: the code generator we use pulls the latest OpenAPI +# spec from the given branch. The spec may contain API changes that aren't +# documented in the Kubernetes release notes. Until the code generator pulls +# the spec from a tag instead of the branch, we can only collect the release +# notes the next time we generate the client. +function util::kube_changelog::find_latest_patch_version { + local kubernetes_branch=$1 + + # trim "release-" prefix + local version=${kubernetes_branch:8} + local changelog="/tmp/k8s-changelog-$version.md" + curl -s -o $changelog "https://raw.githubusercontent.com/kubernetes/kubernetes/master/CHANGELOG/CHANGELOG-$version.md" + echo $(grep "v$version" $changelog | head -1 | sed 's/- \[//g' | sed 's/\].*//g') + rm -f $changelog +} + +# get_api_changelog gets the API Change release notes in the given Kubernetes +# branch for all versions newer than the given trim version. +function util::kube_changelog::get_api_changelog { + local kubernetes_branch="$1" + local trim_version="$2" + + # trim "release-" prefix + local version=${kubernetes_branch:8} + local changelog="/tmp/k8s-changelog-$version.md" + curl -s -o $changelog "https://raw.githubusercontent.com/kubernetes/kubernetes/master/CHANGELOG/CHANGELOG-$version.md" + + # remove changelog for versions less than or equal to $trim_version + sed -i "/^# $trim_version$/q" $changelog + # ignore section titles and empty lines; add "kubernetes/kubernetes" to links; replace newline with liternal "\n" + release_notes=$(sed -n "/^### API Change/,/^#/{/^#/!p}" $changelog | sed -n "{/^$/!p}" | sed 's/(\[\#/(\[kubernetes\/kubernetes\#/g' | sed ':a;N;$!ba;s/\n/\\n/g') + rm -f $changelog + echo "$release_notes" +}