diff --git a/.gitattributes b/.gitattributes index 586fd1ab5..ab2435f87 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,9 @@ licenses-binary/* export-ignore LICENSE-binary export-ignore NOTICE-binary export-ignore +*.bat text eol=crlf +*.cmd text eol=crlf +*.java text eol=lf +*.scala text eol=lf +*.xml text eol=lf +*.py text eol=lf diff --git a/build/release/release.sh b/build/release/release.sh new file mode 100755 index 000000000..7e098adf1 --- /dev/null +++ b/build/release/release.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -o pipefail +set -e +set -x + +ASF_USERNAME=${ASF_USERNAME:?"ASF_USERNAME is required"} +ASF_PASSWORD=${ASF_PASSWORD:?"ASF_PASSWORD is required"} +RELEASE_VERSION=${RELEASE_VERSION:?"RELEASE_VERSION is required"} +RELEASE_RC_NO=${RELEASE_RC_NO:?"RELEASE_RC_NO is required"} + +exit_with_usage() { + local NAME=$(basename $0) + cat << EOF +Usage: $NAME + +Top level targets are: + publish: Publish tarballs to SVN staging repository and jars to Nexus staging repository + finalize: Finalize the release after an RC passes vote + +All other inputs are environment variables + +RELEASE_VERSION - Release version, must match pom.xml and not be SNAPSHOT (e.g. 1.3.0-incubating) +RELEASE_RC_NO - Release RC number, (e.g. 0) + +ASF_USERNAME - Username of ASF committer account +ASF_PASSWORD - Password of ASF committer account +EOF + exit 1 +} + +if [[ ${RELEASE_VERSION} =~ .*-SNAPSHOT ]]; then + echo "Can not release a SNAPSHOT version: ${RELEASE_VERSION}" + exit_with_usage + exit 1 +fi + +RELEASE_TAG="v${RELEASE_VERSION}-rc${RELEASE_RC_NO}" + +SVN_STAGING_REPO="https://dist.apache.org/repos/dist/dev/incubator/kyuubi" +SVN_RELEASE_REPO="https://dist.apache.org/repos/dist/release/incubator/kyuubi" + +KYUUBI_DIR="$(cd "$(dirname "$0")"/../..; pwd)" +RELEASE_DIR="${KYUUBI_DIR}/work/release" +SVN_STAGING_DIR="${KYUUBI_DIR}/work/svn-dev" +SVN_RELEASE_DIR="${KYUUBI_DIR}/work/svn-release" + +package() { + SKIP_GPG="false" RELEASE_VERSION="${RELEASE_VERSION}" $KYUUBI_DIR/build/release/create-package.sh source + SKIP_GPG="false" RELEASE_VERSION="${RELEASE_VERSION}" $KYUUBI_DIR/build/release/create-package.sh binary +} + +upload_svn_staging() { + svn checkout --depth=empty "${SVN_STAGING_REPO}" "${SVN_STAGING_DIR}" + mkdir -p "${SVN_STAGING_DIR}/${RELEASE_TAG}" + rm -f "${SVN_STAGING_DIR}/${RELEASE_TAG}/*" + + SRC_TGZ_FILE="kyuubi-${RELEASE_VERSION}-source.tgz" + BIN_TGZ_FILE="kyuubi-${RELEASE_VERSION}-bin.tgz" + + echo "Copying release tarballs" + cp "${RELEASE_DIR}/${SRC_TGZ_FILE}" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${SRC_TGZ_FILE}" + cp "${RELEASE_DIR}/${SRC_TGZ_FILE}.asc" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${SRC_TGZ_FILE}.asc" + cp "${RELEASE_DIR}/${SRC_TGZ_FILE}.sha512" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${SRC_TGZ_FILE}.sha512" + cp "${RELEASE_DIR}/${BIN_TGZ_FILE}" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${BIN_TGZ_FILE}" + cp "${RELEASE_DIR}/${BIN_TGZ_FILE}.asc" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${BIN_TGZ_FILE}.asc" + cp "${RELEASE_DIR}/${BIN_TGZ_FILE}.sha512" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${BIN_TGZ_FILE}.sha512" + + svn add "${SVN_STAGING_DIR}/${RELEASE_TAG}" + + echo "Uploading release tarballs to ${SVN_STAGING_DIR}/${RELEASE_TAG}" + ( + cd "${SVN_STAGING_DIR}" && \ + svn commit --username "${ASF_USERNAME}" --password "${ASF_PASSWORD}" --message "Apache Kyuubi ${RELEASE_TAG}" + ) + echo "Kyuubi tarballs uploaded" +} + +upload_nexus_staging() { + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided \ + -s "${KYUUBI_DIR}/build/release/asf-settings.xml" +} + +finalize_svn() { + echo "Moving Kyuubi tarballs to the release directory" + svn mv --username "${ASF_USERNAME}" --password "${ASF_PASSWORD}" --no-auth-cache \ + --message"Apache Kyuubi ${RELEASE_VERSION}" \ + "${SVN_STAGING_DIR}/${RELEASE_TAG}" "${SVN_RELEASE_REPO}/kyuubi-${RELEASE_VERSION}" + echo "Kyuubi tarballs moved" + + echo "Sync'ing KEYS" + svn checkout --depth=files "${SVN_RELEASE_REPO}" "${SVN_RELEASE_DIR}" + curl "$SVN_STAGING_REPO/KEYS" > "${SVN_RELEASE_DIR}/KEYS" + svn add "${SVN_RELEASE_DIR}/KEYS" + ( + cd "${SVN_RELEASE_DIR}" && \ + svn commit --username "${ASF_USERNAME}" --password "${ASF_PASSWORD}" --message "Update KEYS" + ) + echo "KEYS sync'ed" +} + +if [[ "$1" == "publish" ]]; then + package + upload_svn_staging + upload_nexus_staging + exit 0 +fi + +if [[ "$1" == "finalize" ]]; then + echo "THIS STEP IS IRREVERSIBLE! Make sure the vote has passed and you pick the right RC to finalize." + read -p "You must be a PMC member to run this step. Continue? [y/N] " ANSWER + if [ "$ANSWER" != "y" ]; then + echo "Exiting." + exit 1 + fi + + finalize_svn + exit 0 +fi + +exit_with_usage diff --git a/build/release/vote.tmpl b/build/release/vote.tmpl new file mode 100644 index 000000000..297188e0a --- /dev/null +++ b/build/release/vote.tmpl @@ -0,0 +1,52 @@ +Please vote on releasing the following candidate as Apache Kyuubi (Incubating) version {version}. + +The vote is open until {deadline} and passes if a majority +1 (P)PMC votes are cast, +with a minimum of 3 +1 votes. + +[ ] +1 Release this package as Apache Kyuubi (Incubating) {version} +[ ] -1 Do not release this package because ... + +To learn more about Apache Kyuubi (Incubating), please see https://kyuubi.apache.org/ + +The tag to be voted on is {tag} (commit {tag_commit}): +https://github.com/apache/incubator-kyuubi/tree/{tag} + +The release files, including signatures, digests, etc. can be found at: +https://dist.apache.org/repos/dist/dev/incubator/kyuubi/{tag} + +Signatures used for Kyuubi RCs can be found in this file: +https://dist.apache.org/repos/dist/dev/incubator/kyuubi/KEYS + +The staging repository for this release can be found at: +https://repository.apache.org/content/repositories/orgapachekyuubi-{repo_id}/ + +FAQ + +================================= +How can I help test this release? +================================= + +If you are a Kyuubi user, you can help us test this release by taking +an existing Kyuubi workload and running on this release candidate, then +reporting any regressions. + +In the Java/Scala, you can add the staging repository to your projects resolvers +and test with the RC (make sure to clean up the artifact cache before/after so +you don't end up building with a out of date RC going forward). + +============================================================== +What should happen to GitHub issues still targeting {version}? +============================================================== + +Committers should look at those and triage. Extremely important bug fixes, +documentation, and API tweaks that impact compatibility should be worked on +immediately. Everything else please retarget to an appropriate release. + +======================= +But my bug isn't fixed? +======================= + +In order to make timely releases, we will typically not hold the release unless +the bug in question is a regression from the previous release. That being said, +if there is something which is a regression that has not been correctly targeted +please ping me or a committer to help target the issue. diff --git a/docs/Makefile b/docs/Makefile index d4bb2cbb9..f596e0968 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# # Minimal makefile for Sphinx documentation # diff --git a/docs/community/index.rst b/docs/community/index.rst index 9e5d72cf5..09c26ef07 100644 --- a/docs/community/index.rst +++ b/docs/community/index.rst @@ -12,4 +12,5 @@ Community collaborators contributions improvement-proposals + release badges diff --git a/docs/community/release.md b/docs/community/release.md new file mode 100644 index 000000000..4c9f48f52 --- /dev/null +++ b/docs/community/release.md @@ -0,0 +1,194 @@ +Kyuubi Release Guide +=== + +## Introduction +The Apache Kyuubi (Incubating) project periodically declares and publishes releases. A release is one or more packages +of the project artifact(s) that are approved for general public distribution and use. They may come with various +degrees of caveat regarding their perceived quality and potential for change, such as "alpha", "beta", "incubating", +"stable", etc. + +The Kyuubi community treats releases with great importance. They are a public face of the project and most users +interact with the project only through the releases. Releases are signed off by the entire Kyuubi community in a +public vote. + +Each release is executed by a Release Manager, who is selected among the Kyuubi committers. This document describes +the process that the Release Manager follows to perform a release. Any changes to this process should be discussed +and adopted on the [dev mailing list](mailto:dev@kyuubi.apache.org). + +Please remember that publishing software has legal consequences. This guide complements the foundation-wide +[Product Release Policy](https://www.apache.org/dev/release.html) and +[Release Distribution Policy](https://www.apache.org/dev/release-distribution). + +### Overview + +![release process](../imgs/release/release-process.png) + +The release process consists of several steps: + +1. Decide to release +2. Prepare for the release +3. Cut branch iff for __major__ release +4. Build a release candidate +5. Vote on the release candidate +6. If necessary, fix any issues and go back to step 3. +7. Finalize the release +8. Promote the release + +## Decide to release + +Deciding to release and selecting a Release Manager is the first step of the release process. This is a consensus-based +decision of the entire community. + +Anybody can propose a release on the [dev mailing list](mailto:dev@kyuubi.apache.org), giving a solid argument and +nominating a committer as the Release Manager (including themselves). There’s no formal process, no vote requirements, +and no timing requirements. Any objections should be resolved by consensus before starting the release. + +In general, the community prefers to have a rotating set of 1-2 Release Managers. Keeping a small core set of managers +allows enough people to build expertise in this area and improve processes over time, without Release Managers needing +to re-learn the processes for each release. That said, if you are a committer interested in serving the community in +this way, please reach out to the community on the [dev mailing list](mailto:dev@kyuubi.apache.org). + +### Checklist to proceed to the next step + +1. Community agrees to release +2. Community selects a Release Manager + +## Prepare for the release + +Before your first release, you should perform one-time configuration steps. This will set up your security keys for +signing the release and access to various release repositories. + +### One-time setup instructions + +#### ASF authentication + +The environments `ASF_USERNAME` and `ASF_PASSWORD` have been used in several places and several times in the release +process, you can either one-time set up them in `~/.bashrc` or `~/.zshrc`, or export them in terminal everytime. + +```shell +export ASF_USERNAME= +export ASF_PASSWORD= +``` + +#### Subversion + +Besides on `git`, `svn` is also required for Apache release, please refer to +https://www.apache.org/dev/version-control.html#https-svn for details. + +#### GPG Key + +You need to have a GPG key to sign the release artifacts. Please be aware of the ASF-wide +[release signing guidelines](https://www.apache.org/dev/release-signing.html). If you don’t have a GPG key associated +with your Apache account, please create one according to the guidelines. + +Determine your Apache GPG Key and Key ID, as follows: +```shell +gpg --list-keys --keyid-format SHORT +``` + +This will list your GPG keys. One of these should reflect your Apache account, for example: +```shell +pub ed25519/ED4E2E5B 2021-07-02 [SC] + 2A145AF50886431A8FA86089F07E6C29ED4E2E5B +uid [ultimate] Cheng Pan +sub cv25519/C7207C04 2021-07-02 [E] +``` +Here, the key ID is the 8-digit hex string in the pub line: ED4E2E5B. + +The last step is to update the KEYS file with your code signing key +https://www.apache.org/dev/openpgp.html#export-public-key + +```shell +svn checkout --depth=files "https://dist.apache.org/repos/dist/dev/incubator/kyuubi" svn-kyuubi +... edit svn-kyuubi/KEYS file +svn commit --username "${ASF_USERNAME}" --password "${ASF_PASSWORD}" --message "Update KEYS" +``` + +## Cut branch iff for major release + +Kyuubi use version pattern `{MAJOR_VERSION}.{MINOR_VERSION}.{PATCH_VERSION}[-{OPTIONAL_SUFFIX}]`, e.g. `1.3.0-incubating`. +__Major Release__ means `MAJOR_VERSION` or `MINOR_VERSION` changed, and __Patch Release__ means `PATCH_VERSION` changed. + +The main step towards preparing a major release is to create a release branch. This is done via standard Git branching +mechanism and should be announced to the community once the branch is created. + +The release branch pattern is `branch-{MAJOR_VERSION}.{MINOR_VERSION}`, e.g. `branch-1.3`. + +After cutting release branch, don't forget bump version in `master` branch. + +## Build a release candidate + +1. Set environment variables. + +```shell +export RELEASE_VERSION= +export RELEASE_RC_NO= +``` + +2. Bump version. + +```shell +build/mvn versions:set -DgenerateBackupPoms=false \ + -DnewVersion="${RELEASE_VERSION}" \ + -Pkubernetes,kyuubi-extension-spark-3-1,spark-block-cleaner,tpcds + +git add . +git commit -m "[RELEASE] Bump ${RELEASE_VERSION}" +``` + +3. Create a git tag for the release candidate. + +The tag pattern is `v${RELEASE_VERSION}-rc${RELEASE_RC_NO}`, e.g. `v1.3.0-incubating-rc0` + +4. Package the release binaries & sources, and upload them to the Apache staging SVN repo. Publish jars to the Apache +staging Maven repo. + +```shell +build/release/releas.sh publish +``` + +## Vote on the release candidate + +The release voting takes place on the Apache Kyuubi (Incubating) developers list (the (P)PMC is voting). + +- If possible, attach a draft of the release notes with the email. +- Recommend represent voting closing time in UTC format. +- Make sure the email is in text format and the links are correct + +Once the vote is done, you should also send out a summary email with the totals, with a subject that looks +something like __[VOTE][RESULT] ....__ + +## Finalize the Release + +__Be Careful!__ + +__THIS STEP IS IRREVERSIBLE so make sure you selected the correct staging repository.__ +__Once you move the artifacts into the release folder, they cannot be removed.__ + +After the vote passes, to upload the binaries to Apache mirrors, you move the binaries from dev directory (this should +be where they are voted) to release directory. This "moving" is the only way you can add stuff to the actual release +directory. (Note: only (P)PMC members can move to release directory) + +Move the sub-directory in "dev" to the corresponding directory in "release". If you've added your signing key to the +KEYS file, also update the release copy. + +```shell +build/release/releas.sh finalize +``` + +Verify that the resources are present in https://www.apache.org/dist/incubator/kyuubi/. It may take a while for them +to be visible. This will be mirrored throughout the Apache network. + +For Maven Central Repository, you can Release from the [Apache Nexus Repository Manager](https://repository.apache.org/). +Log in, open Staging Repositories, find the one voted on, select and click Release and confirm. If successful, it should +show up under https://repository.apache.org/content/repositories/releases/org/apache/kyuubi/ and the same under +https://repository.apache.org/content/groups/maven-staging-group/org/apache/kyuubi/ (look for the correct release version). +After some time this will be sync’d to [Maven Central](https://search.maven.org/) automatically. + +## Promote the release + +### Create an Announcement + +Once everything is working create an announcement on the website and then send an e-mail to the mailing list. + +Enjoy an adult beverage of your choice, and congratulations on making a Kyuubi release. diff --git a/docs/conf.py b/docs/conf.py index e17174e24..35c1c2408 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,21 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full diff --git a/docs/imgs/release/release-process.png b/docs/imgs/release/release-process.png new file mode 100644 index 000000000..bf7602dde Binary files /dev/null and b/docs/imgs/release/release-process.png differ diff --git a/docs/make.bat b/docs/make.bat index 2119f5109..1f441aefc 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,35 +1,52 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd +@ECHO OFF + +rem +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. +rem + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt index af83758c7..6d960748e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + sphinx==3.5.4 sphinx_rtd_theme==0.5.2 sphinx-markdown-tables==0.0.15 diff --git a/pom.xml b/pom.xml index d82b3bfff..59795b70b 100644 --- a/pom.xml +++ b/pom.xml @@ -1505,6 +1505,7 @@ **/*.log **/*.md **/*.iml + **/*.tmpl **/target/** **/out/** **/spark-warehouse/**