2f45e34dc3
### _Why are the changes needed?_ `mvn install` is required because of `kyuubi-hive-jdbc-shaded` ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1571 from pan3793/dist. Closes #1571 c905411c [Cheng Pan] Correct build command in build/dist Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
254 lines
8.0 KiB
Bash
Executable File
254 lines
8.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
#
|
|
# Script to create a binary distribution for easy deploys of Spark.
|
|
# The distribution directory defaults to dist/ but can be overridden below.
|
|
# The distribution contains fat (assembly) jars that include the Scala library,
|
|
# so it is completely self contained.
|
|
# It does not contain source or *.class files.
|
|
|
|
set -o pipefail
|
|
set -e
|
|
|
|
KYUUBI_HOME="$(cd "`dirname "$0"`/.."; pwd)"
|
|
DISTDIR="$KYUUBI_HOME/dist"
|
|
MAKE_TGZ=false
|
|
# TODO: add FLINK_PROVIDED option
|
|
SPARK_PROVIDED=false
|
|
NAME=none
|
|
MVN="$KYUUBI_HOME/build/mvn"
|
|
|
|
function usage {
|
|
set +x
|
|
echo "./build/dist - Tool for making binary distributions of Kyuubi"
|
|
echo ""
|
|
echo "Usage:"
|
|
echo "+--------------------------------------------------------------------------------------+"
|
|
echo "| ./build/dist [--name <custom_name>] [--tgz] [--flink-provided] [--spark-provided] <maven build options> |"
|
|
echo "+--------------------------------------------------------------------------------------+"
|
|
echo "name: - custom binary name, using project version if undefined"
|
|
echo "tgz: - whether to make a whole bundled package"
|
|
echo "flink-provided: - whether to make a package without Flink binary"
|
|
echo "spark-provided: - whether to make a package without Spark binary"
|
|
echo ""
|
|
}
|
|
|
|
function exit_with_usage {
|
|
usage
|
|
exit 1
|
|
}
|
|
|
|
|
|
# Parse arguments
|
|
while (( "$#" )); do
|
|
case $1 in
|
|
--tgz)
|
|
MAKE_TGZ=true
|
|
;;
|
|
--spark-provided)
|
|
SPARK_PROVIDED=true
|
|
;;
|
|
--name)
|
|
NAME="$2"
|
|
shift
|
|
;;
|
|
--help)
|
|
exit_with_usage
|
|
;;
|
|
--*)
|
|
echo "Error: $1 is not supported"
|
|
exit_with_usage
|
|
;;
|
|
-*)
|
|
break
|
|
;;
|
|
*)
|
|
echo "Error: $1 is not supported"
|
|
exit_with_usage
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
# Setup java
|
|
if [[ -z "$JAVA_HOME" ]]; then
|
|
if [[ `command -v java` ]]; then
|
|
# If java is in /usr/bin/java, we want /usr
|
|
JAVA_HOME="$(dirname $(dirname $(which java)))"
|
|
fi
|
|
fi
|
|
|
|
if [[ -z "$JAVA_HOME" ]]; then
|
|
echo "Error: JAVA_HOME is not set, cannot proceed."
|
|
exit -1
|
|
fi
|
|
|
|
echo "JAVA_HOME is set to $JAVA_HOME"
|
|
|
|
if [[ $(command -v git) ]]; then
|
|
GITREV=$(git rev-parse --short HEAD 2>/dev/null || :)
|
|
if [[ ! -z "$GITREV" ]]; then
|
|
GITREVSTRING="(git revision $GITREV)"
|
|
fi
|
|
unset GITREV
|
|
fi
|
|
|
|
VERSION=$("$MVN" help:evaluate -Dexpression=project.version $@ 2>/dev/null\
|
|
| grep -v "INFO"\
|
|
| grep -v "WARNING"\
|
|
| tail -n 1)
|
|
|
|
JAVA_VERSION=$("$MVN" help:evaluate -Dexpression=java.version $@ 2>/dev/null\
|
|
| grep -v "INFO"\
|
|
| grep -v "WARNING"\
|
|
| tail -n 1)
|
|
|
|
SCALA_VERSION=$("$MVN" help:evaluate -Dexpression=scala.binary.version $@ 2>/dev/null\
|
|
| grep -v "INFO"\
|
|
| grep -v "WARNING"\
|
|
| tail -n 1)
|
|
|
|
SPARK_VERSION=$("$MVN" help:evaluate -Dexpression=spark.version $@ 2>/dev/null\
|
|
| grep -v "INFO"\
|
|
| grep -v "WARNING"\
|
|
| tail -n 1)
|
|
|
|
SPARK_HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.binary.version $@ 2>/dev/null\
|
|
| grep -v "INFO"\
|
|
| grep -v "WARNING"\
|
|
| tail -n 1)
|
|
|
|
HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null\
|
|
| grep -v "INFO"\
|
|
| grep -v "WARNING"\
|
|
| tail -n 1)
|
|
|
|
HIVE_VERSION=$("$MVN" help:evaluate -Dexpression=hive.version $@ 2>/dev/null\
|
|
| grep -v "INFO"\
|
|
| grep -v "WARNING"\
|
|
| tail -n 1)
|
|
|
|
echo "Building Kyuubi package of version $VERSION against Spark version - $SPARK_VERSION"
|
|
|
|
SUFFIX="-$NAME"
|
|
if [[ "$NAME" == "none" ]]; then
|
|
if [[ "$SPARK_PROVIDED" == "true" ]]; then
|
|
SUFFIX=""
|
|
else
|
|
SUFFIX="-spark-${SPARK_VERSION:0:3}-hadoop${SPARK_HADOOP_VERSION}"
|
|
fi
|
|
fi
|
|
|
|
if [[ "$MAKE_TGZ" == "true" ]]; then
|
|
echo "Making apache-kyuubi-$VERSION-bin$SUFFIX.tgz"
|
|
else
|
|
echo "Making distribution for Kyuubi $VERSION in '$DISTDIR'..."
|
|
fi
|
|
|
|
MVN_DIST_OPT="-DskipTests"
|
|
if [[ "$SPARK_PROVIDED" == "true" ]]; then
|
|
MVN_DIST_OPT="$MVN_DIST_OPT -Pspark-provided"
|
|
fi
|
|
|
|
BUILD_COMMAND=("$MVN" clean install $MVN_DIST_OPT $@)
|
|
|
|
echo -e "\nBuilding with..."
|
|
echo -e "\$ ${BUILD_COMMAND[@]}\n"
|
|
|
|
"${BUILD_COMMAND[@]}"
|
|
|
|
# Make directories
|
|
rm -rf "$DISTDIR"
|
|
mkdir -p "$DISTDIR/pid"
|
|
mkdir -p "$DISTDIR/logs"
|
|
mkdir -p "$DISTDIR/work"
|
|
mkdir -p "$DISTDIR/externals/engines/spark"
|
|
mkdir -p "$DISTDIR/beeline-jars"
|
|
echo "Kyuubi $VERSION $GITREVSTRING built for" > "$DISTDIR/RELEASE"
|
|
echo "Java $JAVA_VERSION" >> "$DISTDIR/RELEASE"
|
|
echo "Scala $SCALA_VERSION" >> "$DISTDIR/RELEASE"
|
|
echo "Spark $SPARK_VERSION" >> "$DISTDIR/RELEASE"
|
|
echo "Hadoop $SPARK_HADOOP_VERSION" >> "$DISTDIR/RELEASE"
|
|
echo "Hive $HIVE_VERSION" >> "$DISTDIR/RELEASE"
|
|
echo "Build flags: $@" >> "$DISTDIR/RELEASE"
|
|
|
|
# Copy kyuubi server jars
|
|
cp -r "$KYUUBI_HOME/kyuubi-assembly/target/scala-$SCALA_VERSION/jars/" "$DISTDIR/jars"
|
|
|
|
# Copy kyuubi beeline jars
|
|
cp "$KYUUBI_HOME"/kyuubi-hive-beeline/target/*.jar "$DISTDIR/beeline-jars/"
|
|
|
|
# Share the jars between server and beeline to reduce binary size
|
|
cd $DISTDIR/beeline-jars
|
|
for jar in $(ls "$DISTDIR/jars/"); do
|
|
if [[ -f "$DISTDIR/beeline-jars/$jar" ]]; then
|
|
rm "$DISTDIR/beeline-jars/$jar"
|
|
ln -sn "../jars/$jar" "$DISTDIR/beeline-jars/$jar"
|
|
fi
|
|
done
|
|
cd -
|
|
|
|
# Copy spark engines
|
|
cp "$KYUUBI_HOME/externals/kyuubi-spark-sql-engine/target/kyuubi-spark-sql-engine_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/externals/engines/spark"
|
|
|
|
# Copy kyuubi tools
|
|
if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" ]]; then
|
|
mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes"
|
|
mkdir -p "$DISTDIR/tools/spark-block-cleaner/jars"
|
|
cp -r "$KYUUBI_HOME/tools/spark-block-cleaner/kubernetes/" "$DISTDIR/tools/spark-block-cleaner/kubernetes/"
|
|
cp "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/tools/spark-block-cleaner/jars/"
|
|
fi
|
|
|
|
# Copy Kyuubi extension
|
|
SPARK_EXTENSION_VERSIONS=('3-1' '3-2')
|
|
for SPARK_EXTENSION_VERSION in ${SPARK_EXTENSION_VERSIONS[@]}; do
|
|
if [[ -f $"$KYUUBI_HOME/dev/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION/target/kyuubi-extension-spark-${SPARK_EXTENSION_VERSION}_${SCALA_VERSION}-${VERSION}.jar" ]]; then
|
|
mkdir -p "$DISTDIR/extension"
|
|
cp "$KYUUBI_HOME/dev/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION/target/kyuubi-extension-spark-${SPARK_EXTENSION_VERSION}_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/extension"
|
|
fi
|
|
done
|
|
|
|
if [[ "$SPARK_PROVIDED" != "true" ]]; then
|
|
# Copy spark binary dist
|
|
cp -r "$KYUUBI_HOME/externals/kyuubi-download/target/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/" \
|
|
"$DISTDIR/externals/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/"
|
|
fi
|
|
|
|
# Copy license files
|
|
cp "$KYUUBI_HOME/DISCLAIMER" "$DISTDIR/DISCLAIMER"
|
|
if [[ -f $"$KYUUBI_HOME/LICENSE-binary" ]]; then
|
|
cp "$KYUUBI_HOME/LICENSE-binary" "$DISTDIR/LICENSE"
|
|
cp -r "$KYUUBI_HOME/licenses-binary" "$DISTDIR/licenses"
|
|
$KYUUBI_HOME/build/release/collect-licenses.sh "$DISTDIR" "$DISTDIR"
|
|
fi
|
|
|
|
cp -r "$KYUUBI_HOME/bin" "$DISTDIR"
|
|
cp -r "$KYUUBI_HOME/conf" "$DISTDIR"
|
|
cp -r "$KYUUBI_HOME/docker" "$DISTDIR"
|
|
|
|
if [[ "$MAKE_TGZ" == "true" ]]; then
|
|
TARDIR_NAME="apache-kyuubi-$VERSION-bin$SUFFIX"
|
|
TARDIR="$KYUUBI_HOME/$TARDIR_NAME"
|
|
rm -rf "$TARDIR"
|
|
cp -R "$DISTDIR" "$TARDIR"
|
|
tar czf "$TARDIR_NAME.tgz" -C "$KYUUBI_HOME" "$TARDIR_NAME"
|
|
rm -rf "$TARDIR"
|
|
echo "The Kyuubi tarball $TARDIR_NAME.tgz is successfully generated in $KYUUBI_HOME."
|
|
fi
|