kyuubi/build/dist

#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# Script to create a binary distribution for easy deploys of Spark.
# The distribution directory defaults to dist/ but can be overridden below.
# The distribution contains fat (assembly) jars that include the Scala library,
# so it is completely self contained.
# It does not contain source or *.class files.

set -o pipefail
set -e
set -x

KYUUBI_HOME="$(cd "`dirname "$0"`/.."; pwd)"
DISTDIR="$KYUUBI_HOME/dist"
MAKE_TGZ=false
SPARK_PROVIDED=false
NAME=none
MVN="$KYUUBI_HOME/build/mvn"

function usage {
  set +x
  echo "./build/dist - Tool for making binary distributions of Kyuubi"
  echo ""
  echo "Usage:"
  echo "+--------------------------------------------------------------------------------------+"
  echo "| ./build/dist [--name <custom_name>] [--tgz] [--spark-provided] <maven build options> |"
  echo "+--------------------------------------------------------------------------------------+"
  echo "name:           -  custom binary name, using project version if undefined"
  echo "tgz:            -  whether to make a whole bundled package"
  echo "spark-provided: -  whether to make a package without Spark binary"
  echo ""
}

function exit_with_usage {
  usage
  exit 1
}


# Parse arguments
while (( "$#" )); do
  case $1 in
    --tgz)
      MAKE_TGZ=true
      ;;
    --spark-provided)
      SPARK_PROVIDED=true
      ;;
    --name)
      NAME="$2"
      shift
      ;;
    --help)
      exit_with_usage
      ;;
    --*)
      echo "Error: $1 is not supported"
      exit_with_usage
      ;;
    -*)
      break
      ;;
    *)
      echo "Error: $1 is not supported"
      exit_with_usage
      ;;
  esac
  shift
done
# Setup java
if [[ -z "$JAVA_HOME" ]]; then
  if [[ `command -v java` ]]; then
    # If java is in /usr/bin/java, we want /usr
    JAVA_HOME="$(dirname $(dirname $(which java)))"
  fi
fi

if [[ -z "$JAVA_HOME" ]]; then
  echo "Error: JAVA_HOME is not set, cannot proceed."
  exit -1
fi

echo "JAVA_HOME is set to $JAVA_HOME"

if [[ $(command -v git) ]]; then
    GITREV=$(git rev-parse --short HEAD 2>/dev/null || :)
    if [[ ! -z "$GITREV" ]]; then
        GITREVSTRING="(git revision $GITREV)"
    fi
    unset GITREV
fi

VERSION=$("$MVN" help:evaluate -Dexpression=project.version $@ 2>/dev/null\
    | grep -v "INFO"\
    | grep -v "WARNING"\
    | tail -n 1)

JAVA_VERSION=$("$MVN" help:evaluate -Dexpression=java.version $@ 2>/dev/null\
    | grep -v "INFO"\
    | grep -v "WARNING"\
    | tail -n 1)

SCALA_VERSION=$("$MVN" help:evaluate -Dexpression=scala.binary.version $@ 2>/dev/null\
    | grep -v "INFO"\
    | grep -v "WARNING"\
    | tail -n 1)

SPARK_VERSION=$("$MVN" help:evaluate -Dexpression=spark.version $@ 2>/dev/null\
    | grep -v "INFO"\
    | grep -v "WARNING"\
    | tail -n 1)

SPARK_HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.binary.version $@ 2>/dev/null\
    | grep -v "INFO"\
    | grep -v "WARNING"\
    | tail -n 1)

HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null\
    | grep -v "INFO"\
    | grep -v "WARNING"\
    | tail -n 1)

HIVE_VERSION=$("$MVN" help:evaluate -Dexpression=hive.version $@ 2>/dev/null\
    | grep -v "INFO"\
    | grep -v "WARNING"\
    | tail -n 1)

echo "Building Kyuubi package of version $VERSION against Spark version - $SPARK_VERSION"

if [[ "$NAME" == "none" ]]; then
  SPARK_HADOOP_VERSION_SUFFIX="-hadoop${SPARK_HADOOP_VERSION}"

  if [[ "$SPARK_PROVIDED" == "true" ]]; then
    NAME="without-spark"
  else
    NAME="spark-"${SPARK_VERSION:0:3}$SPARK_HADOOP_VERSION_SUFFIX
  fi
fi

if [[ "$MAKE_TGZ" == "true" ]]; then
  echo "Making kyuubi-$VERSION-bin-$NAME.tar.gz"
else
  echo "Making distribution for Kyuubi $VERSION named $NAME in '$DISTDIR'..."
fi

MVN_DIST_OPT="-DskipTests"
if [[ "$SPARK_PROVIDED" == "true" ]]; then
  MVN_DIST_OPT="$MVN_DIST_OPT -Pspark-provided"
fi

BUILD_COMMAND=("$MVN" -T 1C clean package $MVN_DIST_OPT $@)

echo -e "\nBuilding with..."
echo -e "\$ ${BUILD_COMMAND[@]}\n"

"${BUILD_COMMAND[@]}"

# Make directories
rm -rf "$DISTDIR"
mkdir -p "$DISTDIR/pid"
mkdir -p "$DISTDIR/logs"
mkdir -p "$DISTDIR/work"
mkdir -p "$DISTDIR/externals/engines/spark"
echo "Kyuubi $VERSION $GITREVSTRING built for" > "$DISTDIR/RELEASE"
echo "Java $JAVA_VERSION" >> "$DISTDIR/RELEASE"
echo "Scala $SCALA_VERSION" >> "$DISTDIR/RELEASE"
echo "Spark $SPARK_VERSION" >> "$DISTDIR/RELEASE"
echo "Hadoop $SPARK_HADOOP_VERSION" >> "$DISTDIR/RELEASE"
echo "Hive $HIVE_VERSION" >> "$DISTDIR/RELEASE"
echo "Build flags: $@" >> "$DISTDIR/RELEASE"

# Copy kyuubi server jars
cp -r "$KYUUBI_HOME/kyuubi-assembly/target/scala-$SCALA_VERSION/jars/" "$DISTDIR/jars"

# Copy spark engines
cp "$KYUUBI_HOME/externals/kyuubi-spark-sql-engine/target/kyuubi-spark-sql-engine_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/externals/engines/spark"

# Copy kyuubi tools
if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" ]]; then
  mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes"
  mkdir -p "$DISTDIR/tools/spark-block-cleaner/jars"
  cp -r "$KYUUBI_HOME/tools/spark-block-cleaner/kubernetes/" "$DISTDIR/tools/spark-block-cleaner/"
  cp "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/tools/spark-block-cleaner/jars/"
fi

# Copy Kyuubi extension
SPARK_EXTENSION_VERSIONS=('3-1')
for SPARK_EXTENSION_VERSION in ${SPARK_EXTENSION_VERSIONS[@]}; do
  if [[ -f $"$KYUUBI_HOME/dev/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION/target/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION_${SCALA_VERSION}-${VERSION}.jar" ]]; then
    mkdir -p "$DISTDIR/extension"
    cp "$KYUUBI_HOME/dev/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION/target/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/extension"
  fi
done

if [[ "$SPARK_PROVIDED" != "true" ]]; then
  # Copy spark binary dist
  cp -r "$KYUUBI_HOME/externals/kyuubi-download/target/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/" \
        "$DISTDIR/externals/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/"
fi

# Copy license files
if [ -e "$KYUUBI_HOME/LICENSE-binary" ]; then
  cp "$KYUUBI_HOME/LICENSE-binary" "$DISTDIR/LICENSE"
  cp -r "$KYUUBI_HOME/licenses-binary" "$DISTDIR/licenses"
else
  echo "Skipping copying LICENSE files"
fi

cp -r "$KYUUBI_HOME/bin" "$DISTDIR"
cp -r "$KYUUBI_HOME/conf" "$DISTDIR"
cp -r "$KYUUBI_HOME/docker" "$DISTDIR"

if [[ "$MAKE_TGZ" == "true" ]]; then
  TARDIR_NAME=kyuubi-$VERSION-bin-$NAME
  TARDIR="$KYUUBI_HOME/$TARDIR_NAME"
  rm -rf "$TARDIR"
  cp -r "$DISTDIR" "$TARDIR"
  tar czf "$TARDIR_NAME.tar.gz" -C "$KYUUBI_HOME" "$TARDIR_NAME"
  rm -rf "$TARDIR"
fi