apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #874] [ASF] ASF Publish

Browse Source 
<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->
- Add release script
- Add release guide
- Add license header in some files

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request

The release script has not been tested yet.

Closes #874 from pan3793/asf-pub.

Closes #874

5e798a96 [Cheng Pan] LICENSE header
825f2a84 [Cheng Pan] Fix rat
aecc3e6b [Cheng Pan] release doc
a55503c5 [Cheng Pan] Kyuubi Release Guide
2f1a25a2 [Cheng Pan] Release Guide
c5a40c24 [Cheng Pan] Release script

Authored-by: Cheng Pan <chengpan@apache.org>
Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Cheng Pan 2021-08-16 11:48:21 +08:00
parent 9efa78b65e
commit c0606713e7
No known key found for this signature in database
GPG Key ID: F07E6C29ED4E2E5B
11 changed files with 499 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitattributes vendored
View File

@ -1,3 +1,9 @@
licenses-binary/* export-ignore
LICENSE-binary export-ignore
NOTICE-binary export-ignore
*.bat text eol=crlf
*.cmd text eol=crlf
*.java text eol=lf
*.scala text eol=lf
*.xml text eol=lf
*.py text eol=lf

138
build/release/release.sh Executable file
View File

@ -0,0 +1,138 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -o pipefail
set -e
set -x
ASF_USERNAME=${ASF_USERNAME:?"ASF_USERNAME is required"}
ASF_PASSWORD=${ASF_PASSWORD:?"ASF_PASSWORD is required"}
RELEASE_VERSION=${RELEASE_VERSION:?"RELEASE_VERSION is required"}
RELEASE_RC_NO=${RELEASE_RC_NO:?"RELEASE_RC_NO is required"}
exit_with_usage() {
local NAME=$(basename $0)
cat << EOF
Usage: $NAME <publish|finalize>
Top level targets are:
publish: Publish tarballs to SVN staging repository and jars to Nexus staging repository
finalize: Finalize the release after an RC passes vote
All other inputs are environment variables
RELEASE_VERSION - Release version, must match pom.xml and not be SNAPSHOT (e.g. 1.3.0-incubating)
RELEASE_RC_NO - Release RC number, (e.g. 0)
ASF_USERNAME - Username of ASF committer account
ASF_PASSWORD - Password of ASF committer account
EOF
exit 1
}
if [[ ${RELEASE_VERSION} =~ .*-SNAPSHOT ]]; then
echo "Can not release a SNAPSHOT version: ${RELEASE_VERSION}"
exit_with_usage
exit 1
fi
RELEASE_TAG="v${RELEASE_VERSION}-rc${RELEASE_RC_NO}"
SVN_STAGING_REPO="https://dist.apache.org/repos/dist/dev/incubator/kyuubi"
SVN_RELEASE_REPO="https://dist.apache.org/repos/dist/release/incubator/kyuubi"
KYUUBI_DIR="$(cd "$(dirname "$0")"/../..; pwd)"
RELEASE_DIR="${KYUUBI_DIR}/work/release"
SVN_STAGING_DIR="${KYUUBI_DIR}/work/svn-dev"
SVN_RELEASE_DIR="${KYUUBI_DIR}/work/svn-release"
package() {
SKIP_GPG="false" RELEASE_VERSION="${RELEASE_VERSION}" $KYUUBI_DIR/build/release/create-package.sh source
SKIP_GPG="false" RELEASE_VERSION="${RELEASE_VERSION}" $KYUUBI_DIR/build/release/create-package.sh binary
}
upload_svn_staging() {
svn checkout --depth=empty "${SVN_STAGING_REPO}" "${SVN_STAGING_DIR}"
mkdir -p "${SVN_STAGING_DIR}/${RELEASE_TAG}"
rm -f "${SVN_STAGING_DIR}/${RELEASE_TAG}/*"
SRC_TGZ_FILE="kyuubi-${RELEASE_VERSION}-source.tgz"
BIN_TGZ_FILE="kyuubi-${RELEASE_VERSION}-bin.tgz"
echo "Copying release tarballs"
cp "${RELEASE_DIR}/${SRC_TGZ_FILE}" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${SRC_TGZ_FILE}"
cp "${RELEASE_DIR}/${SRC_TGZ_FILE}.asc" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${SRC_TGZ_FILE}.asc"
cp "${RELEASE_DIR}/${SRC_TGZ_FILE}.sha512" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${SRC_TGZ_FILE}.sha512"
cp "${RELEASE_DIR}/${BIN_TGZ_FILE}" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${BIN_TGZ_FILE}"
cp "${RELEASE_DIR}/${BIN_TGZ_FILE}.asc" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${BIN_TGZ_FILE}.asc"
cp "${RELEASE_DIR}/${BIN_TGZ_FILE}.sha512" "${SVN_STAGING_DIR}/${RELEASE_TAG}/${BIN_TGZ_FILE}.sha512"
svn add "${SVN_STAGING_DIR}/${RELEASE_TAG}"
echo "Uploading release tarballs to ${SVN_STAGING_DIR}/${RELEASE_TAG}"
(
cd "${SVN_STAGING_DIR}" && \
svn commit --username "${ASF_USERNAME}" --password "${ASF_PASSWORD}" --message "Apache Kyuubi ${RELEASE_TAG}"
)
echo "Kyuubi tarballs uploaded"
}
upload_nexus_staging() {
${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided \
-s "${KYUUBI_DIR}/build/release/asf-settings.xml"
}
finalize_svn() {
echo "Moving Kyuubi tarballs to the release directory"
svn mv --username "${ASF_USERNAME}" --password "${ASF_PASSWORD}" --no-auth-cache \
--message"Apache Kyuubi ${RELEASE_VERSION}" \
"${SVN_STAGING_DIR}/${RELEASE_TAG}" "${SVN_RELEASE_REPO}/kyuubi-${RELEASE_VERSION}"
echo "Kyuubi tarballs moved"
echo "Sync'ing KEYS"
svn checkout --depth=files "${SVN_RELEASE_REPO}" "${SVN_RELEASE_DIR}"
curl "$SVN_STAGING_REPO/KEYS" > "${SVN_RELEASE_DIR}/KEYS"
svn add "${SVN_RELEASE_DIR}/KEYS"
(
cd "${SVN_RELEASE_DIR}" && \
svn commit --username "${ASF_USERNAME}" --password "${ASF_PASSWORD}" --message "Update KEYS"
)
echo "KEYS sync'ed"
}
if [[ "$1" == "publish" ]]; then
package
upload_svn_staging
upload_nexus_staging
exit 0
fi
if [[ "$1" == "finalize" ]]; then
echo "THIS STEP IS IRREVERSIBLE! Make sure the vote has passed and you pick the right RC to finalize."
read -p "You must be a PMC member to run this step. Continue? [y/N] " ANSWER
if [ "$ANSWER" != "y" ]; then
echo "Exiting."
exit 1
fi
finalize_svn
exit 0
fi
exit_with_usage

52
build/release/vote.tmpl Normal file
View File

@ -0,0 +1,52 @@
Please vote on releasing the following candidate as Apache Kyuubi (Incubating) version {version}.
The vote is open until {deadline} and passes if a majority +1 (P)PMC votes are cast,
with a minimum of 3 +1 votes.
[ ] +1 Release this package as Apache Kyuubi (Incubating) {version}
[ ] -1 Do not release this package because ...
To learn more about Apache Kyuubi (Incubating), please see https://kyuubi.apache.org/
The tag to be voted on is {tag} (commit {tag_commit}):
https://github.com/apache/incubator-kyuubi/tree/{tag}
The release files, including signatures, digests, etc. can be found at:
https://dist.apache.org/repos/dist/dev/incubator/kyuubi/{tag}
Signatures used for Kyuubi RCs can be found in this file:
https://dist.apache.org/repos/dist/dev/incubator/kyuubi/KEYS
The staging repository for this release can be found at:
https://repository.apache.org/content/repositories/orgapachekyuubi-{repo_id}/
FAQ
=================================
How can I help test this release?
=================================
If you are a Kyuubi user, you can help us test this release by taking
an existing Kyuubi workload and running on this release candidate, then
reporting any regressions.
In the Java/Scala, you can add the staging repository to your projects resolvers
and test with the RC (make sure to clean up the artifact cache before/after so
you don't end up building with a out of date RC going forward).
==============================================================
What should happen to GitHub issues still targeting {version}?
==============================================================
Committers should look at those and triage. Extremely important bug fixes,
documentation, and API tweaks that impact compatibility should be worked on
immediately. Everything else please retarget to an appropriate release.
=======================
But my bug isn't fixed?
=======================
In order to make timely releases, we will typically not hold the release unless
the bug in question is a regression from the previous release. That being said,
if there is something which is a regression that has not been correctly targeted
please ping me or a committer to help target the issue.

18
docs/Makefile
View File

@ -1,3 +1,21 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Minimal makefile for Sphinx documentation
#

1
docs/community/index.rst
View File

@ -12,4 +12,5 @@ Community
collaborators
contributions
improvement-proposals
release
badges

194
docs/community/release.md Normal file
View File

@ -0,0 +1,194 @@
Kyuubi Release Guide
===
## Introduction
The Apache Kyuubi (Incubating) project periodically declares and publishes releases. A release is one or more packages
of the project artifact(s) that are approved for general public distribution and use. They may come with various
degrees of caveat regarding their perceived quality and potential for change, such as "alpha", "beta", "incubating",
"stable", etc.
The Kyuubi community treats releases with great importance. They are a public face of the project and most users
interact with the project only through the releases. Releases are signed off by the entire Kyuubi community in a
public vote.
Each release is executed by a Release Manager, who is selected among the Kyuubi committers. This document describes
the process that the Release Manager follows to perform a release. Any changes to this process should be discussed
and adopted on the [dev mailing list](mailto:dev@kyuubi.apache.org).
Please remember that publishing software has legal consequences. This guide complements the foundation-wide
[Product Release Policy](https://www.apache.org/dev/release.html) and
[Release Distribution Policy](https://www.apache.org/dev/release-distribution).
### Overview
![release process](../imgs/release/release-process.png)
The release process consists of several steps:
1. Decide to release
2. Prepare for the release
3. Cut branch iff for __major__ release
4. Build a release candidate
5. Vote on the release candidate
6. If necessary, fix any issues and go back to step 3.
7. Finalize the release
8. Promote the release
## Decide to release
Deciding to release and selecting a Release Manager is the first step of the release process. This is a consensus-based
decision of the entire community.
Anybody can propose a release on the [dev mailing list](mailto:dev@kyuubi.apache.org), giving a solid argument and
nominating a committer as the Release Manager (including themselves). There’s no formal process, no vote requirements,
and no timing requirements. Any objections should be resolved by consensus before starting the release.
In general, the community prefers to have a rotating set of 1-2 Release Managers. Keeping a small core set of managers
allows enough people to build expertise in this area and improve processes over time, without Release Managers needing
to re-learn the processes for each release. That said, if you are a committer interested in serving the community in
this way, please reach out to the community on the [dev mailing list](mailto:dev@kyuubi.apache.org).
### Checklist to proceed to the next step
1. Community agrees to release
2. Community selects a Release Manager
## Prepare for the release
Before your first release, you should perform one-time configuration steps. This will set up your security keys for
signing the release and access to various release repositories.
### One-time setup instructions
#### ASF authentication
The environments `ASF_USERNAME` and `ASF_PASSWORD` have been used in several places and several times in the release
process, you can either one-time set up them in `~/.bashrc` or `~/.zshrc`, or export them in terminal everytime.
```shell
export ASF_USERNAME=<your apache username>
export ASF_PASSWORD=<your apache password>
```
#### Subversion
Besides on `git`, `svn` is also required for Apache release, please refer to
https://www.apache.org/dev/version-control.html#https-svn for details.
#### GPG Key
You need to have a GPG key to sign the release artifacts. Please be aware of the ASF-wide
[release signing guidelines](https://www.apache.org/dev/release-signing.html). If you don’t have a GPG key associated
with your Apache account, please create one according to the guidelines.
Determine your Apache GPG Key and Key ID, as follows:
```shell
gpg --list-keys --keyid-format SHORT
```
This will list your GPG keys. One of these should reflect your Apache account, for example:
```shell
pub ed25519/ED4E2E5B 2021-07-02 [SC]
2A145AF50886431A8FA86089F07E6C29ED4E2E5B
uid [ultimate] Cheng Pan <chengpan@apache.org>
sub cv25519/C7207C04 2021-07-02 [E]
```
Here, the key ID is the 8-digit hex string in the pub line: ED4E2E5B.
The last step is to update the KEYS file with your code signing key
https://www.apache.org/dev/openpgp.html#export-public-key
```shell
svn checkout --depth=files "https://dist.apache.org/repos/dist/dev/incubator/kyuubi" svn-kyuubi
... edit svn-kyuubi/KEYS file
svn commit --username "${ASF_USERNAME}" --password "${ASF_PASSWORD}" --message "Update KEYS"
```
## Cut branch iff for major release
Kyuubi use version pattern `{MAJOR_VERSION}.{MINOR_VERSION}.{PATCH_VERSION}[-{OPTIONAL_SUFFIX}]`, e.g. `1.3.0-incubating`.
__Major Release__ means `MAJOR_VERSION` or `MINOR_VERSION` changed, and __Patch Release__ means `PATCH_VERSION` changed.
The main step towards preparing a major release is to create a release branch. This is done via standard Git branching
mechanism and should be announced to the community once the branch is created.
The release branch pattern is `branch-{MAJOR_VERSION}.{MINOR_VERSION}`, e.g. `branch-1.3`.
After cutting release branch, don't forget bump version in `master` branch.
## Build a release candidate
1. Set environment variables.
```shell
export RELEASE_VERSION=<release version, e.g. 1.3.0-incubating>
export RELEASE_RC_NO=<RC number, e.g. 0>
```
2. Bump version.
```shell
build/mvn versions:set -DgenerateBackupPoms=false \
-DnewVersion="${RELEASE_VERSION}" \
-Pkubernetes,kyuubi-extension-spark-3-1,spark-block-cleaner,tpcds
git add .
git commit -m "[RELEASE] Bump ${RELEASE_VERSION}"
```
3. Create a git tag for the release candidate.
The tag pattern is `v${RELEASE_VERSION}-rc${RELEASE_RC_NO}`, e.g. `v1.3.0-incubating-rc0`
4. Package the release binaries & sources, and upload them to the Apache staging SVN repo. Publish jars to the Apache
staging Maven repo.
```shell
build/release/releas.sh publish
```
## Vote on the release candidate
The release voting takes place on the Apache Kyuubi (Incubating) developers list (the (P)PMC is voting).
- If possible, attach a draft of the release notes with the email.
- Recommend represent voting closing time in UTC format.
- Make sure the email is in text format and the links are correct
Once the vote is done, you should also send out a summary email with the totals, with a subject that looks
something like __[VOTE][RESULT] ....__
## Finalize the Release
__Be Careful!__
__THIS STEP IS IRREVERSIBLE so make sure you selected the correct staging repository.__
__Once you move the artifacts into the release folder, they cannot be removed.__
After the vote passes, to upload the binaries to Apache mirrors, you move the binaries from dev directory (this should
be where they are voted) to release directory. This "moving" is the only way you can add stuff to the actual release
directory. (Note: only (P)PMC members can move to release directory)
Move the sub-directory in "dev" to the corresponding directory in "release". If you've added your signing key to the
KEYS file, also update the release copy.
```shell
build/release/releas.sh finalize
```
Verify that the resources are present in https://www.apache.org/dist/incubator/kyuubi/. It may take a while for them
to be visible. This will be mirrored throughout the Apache network.
For Maven Central Repository, you can Release from the [Apache Nexus Repository Manager](https://repository.apache.org/).
Log in, open Staging Repositories, find the one voted on, select and click Release and confirm. If successful, it should
show up under https://repository.apache.org/content/repositories/releases/org/apache/kyuubi/ and the same under
https://repository.apache.org/content/groups/maven-staging-group/org/apache/kyuubi/ (look for the correct release version).
After some time this will be sync’d to [Maven Central](https://search.maven.org/) automatically.
## Promote the release
### Create an Announcement
Once everything is working create an announcement on the website and then send an e-mail to the mailing list.
Enjoy an adult beverage of your choice, and congratulations on making a Kyuubi release.

18
docs/conf.py
View File

@ -1,3 +1,21 @@
#!/usr/bin/env python3
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full

BIN
docs/imgs/release/release-process.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 355 KiB

87
docs/make.bat
View File

@ -1,35 +1,52 @@
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=.
set BUILDDIR=_build
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
:end
popd
@ECHO OFF
rem
rem Licensed to the Apache Software Foundation (ASF) under one or more
rem contributor license agreements. See the NOTICE file distributed with
rem this work for additional information regarding copyright ownership.
rem The ASF licenses this file to You under the Apache License, Version 2.0
rem (the "License"); you may not use this file except in compliance with
rem the License. You may obtain a copy of the License at
rem
rem http://www.apache.org/licenses/LICENSE-2.0
rem
rem Unless required by applicable law or agreed to in writing, software
rem distributed under the License is distributed on an "AS IS" BASIS,
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
rem See the License for the specific language governing permissions and
rem limitations under the License.
rem
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=.
set BUILDDIR=_build
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
:end
popd

19
docs/requirements.txt
View File

@ -1,3 +1,22 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
sphinx==3.5.4
sphinx_rtd_theme==0.5.2
sphinx-markdown-tables==0.0.15

1
pom.xml
View File

@ -1505,6 +1505,7 @@
<exclude>**/*.log</exclude>
<exclude>**/*.md</exclude>
<exclude>**/*.iml</exclude>
<exclude>**/*.tmpl</exclude>
<exclude>**/target/**</exclude>
<exclude>**/out/**</exclude>
<exclude>**/spark-warehouse/**</exclude>