diff --git a/bin/kyuubi.sh b/bin/kyuubi.sh new file mode 100755 index 000000000..4ea9a8f2d --- /dev/null +++ b/bin/kyuubi.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## Kyuubi Server Main Entrance +CLASS="org.apache.kyuubi.server.KyuubiServer" + +function usage { + echo "Usage: ./bin/kyuubi.sh (start|stop|status)" +} + +if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then + usage + exit 0 +fi + +function kyuubi_rotate_log() { + log=$1; + + if [[ -z ${KYUUBI_MAX_LOG_FILES} ]]; then + num=5 + elif [[ ${KYUUBI_MAX_LOG_FILES} -gt 0 ]]; then + num=${KYUUBI_MAX_LOG_FILES} + else + echo "Error: KYUUBI_MAX_LOG_FILES must be a positive number, but got ${KYUUBI_MAX_LOG_FILES}" + exit -1 + fi + + if [ -f "$log" ]; then # rotate logs + while [ ${num} -gt 1 ]; do + prev=`expr ${num} - 1` + [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" + num=${prev} + done + mv "$log" "$log.$num"; + fi +} + +export KYUUBI_HOME="$(cd "`dirname "$0"`"/..; pwd)" + +echo "Starting Kyuubi Server from ${KYUUBI_HOME}" + +. "${KYUUBI_HOME}/bin/load-kyuubi-env.sh" + +if [[ -z ${JAVA_HOME} ]]; then + echo "Error: JAVA_HOME IS NOT SET! CANNOT PROCEED." + exit 1 +fi + +if [[ -z ${SPARK_HOME} ]]; then + echo "Error: SPARK_HOME IS NOT SET! CANNOT PROCEED." >&2 + exit 1 +else + if [[ ! -x "${SPARK_HOME}/bin/spark-submit" ]]; then + echo echo "Error: INVALID SPARK DISTRIBUTION! CANNOT PROCEED." >&2 + exit 1 + fi +fi + +RUNNER="${JAVA_HOME}/bin/java" + +COMMAND="${RUNNER} ${KYUUBI_JAVA_OPTS} -cp" + +## Find the Kyuubi Jar +if [[ -z "$KYUUBI_JAR_DIR" ]]; then + KYUUBI_JAR_DIR="$KYUUBI_HOME/jars" + if [[ ! -d ${KYUUBI_JAR_DIR} ]]; then + echo -e "\nCandidate Kyuubi lib $KYUUBI_JAR_DIR doesn't exist, searching development environment..." + KYUUBI_JAR_DIR="$KYUUBI_HOME/kyuubi-assembly/target/scala-${KYUUBI_SCALA_VERSION}/jars" + fi +fi + +pid="${KYUUBI_PID_DIR}/kyuubi-$USER-$CLASS.pid" + +function start_kyuubi() { + if [[ ! -w ${KYUUBI_PID_DIR} ]]; then + echo "${USER} does not have 'w' permission to ${KYUUBI_PID_DIR}" + exit 1 + fi + + if [[ ! -w ${KYUUBI_LOG_DIR} ]]; then + echo "${USER} does not have 'w' permission to ${KYUUBI_LOG_DIR}" + exit 1 + fi + + if [ -f "$pid" ]; then + TARGET_ID="$(cat "$pid")" + if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then + echo "$CLASS running as process $TARGET_ID Stop it first." + exit 1 + fi + fi + + log="${KYUUBI_LOG_DIR}/kyuubi-$USER-$CLASS-$HOSTNAME.out" + kyuubi_rotate_log ${log} + + KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}" + cmd="${RUNNER} ${KYUUBI_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS" + echo "Starting $CLASS, logging to $log" + nohup nice -n "${KYUUBI_NICENESS:-0}" ${cmd} >> ${log} 2>&1 < /dev/null & + newpid="$!" + + echo "$newpid" > "$pid" + + # Poll for up to 5 seconds for the java process to start + for i in {1..10} + do + if [[ $(ps -p "$newpid" -o comm=) =~ "java" ]]; then + break + fi + sleep 0.5 + done + + sleep 2 + # Check if the process has died; in that case we'll tail the log so the user can see + if [[ ! $(ps -p "$newpid" -o comm=) =~ "java" ]]; then + echo "Failed to launch: ${cmd}" + tail -2 "$log" | sed 's/^/ /' + echo "Full log in $log" + else + echo "Welcome to" + cat ${KYUUBI_HOME}/bin/kyuubi-logo + fi +} + +function stop_kyuubi() { + if [ -f ${pid} ]; then + TARGET_ID="$(cat "$pid")" + if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then + echo "Stopping $CLASS" + kill "$TARGET_ID" && rm -f "$pid" + for i in {1..20} + do + sleep 0.5 + if [[ ! $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then + break + fi + done + + if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then + echo "Failed to stop kyuubi after 10 seconds, try 'kill -9 ${TARGET_ID}' forcefully " + else + cat ${KYUUBI_HOME}/bin/kyuubi-logo + echo "Bye!" + fi + else + echo "no $CLASS to stop" + fi + else + echo "no $CLASS to stop" + fi +} + +function check_kyuubi() { + if [[ -f ${pid} ]]; then + TARGET_ID="$(cat "$pid")" + if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then + echo "Kyuubi is running (pid: $TARGET_ID)" + else + echo "Kyuubi is not running" + fi + else + echo "Kyuubi is not running" + fi + +} +case $1 in + (start | "") + start_kyuubi + ;; + + (stop) + stop_kyuubi + ;; + + (status) + check_kyuubi + ;; + + (*) + usage + ;; +esac diff --git a/bin/load-kyuubi-env.sh b/bin/load-kyuubi-env.sh new file mode 100755 index 000000000..339b51217 --- /dev/null +++ b/bin/load-kyuubi-env.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +export KYUUBI_HOME="${KYUUBI_HOME:-"$(cd "`dirname $0`"/..; pwd)"}" + +export KYUUBI_CONF_DIR="${KYUUBI_CONF_DIR:-"${KYUUBI_HOME}"/conf}" + +KYUUBI_ENV_SH="${KYUUBI_CONF_DIR}"/kyuubi-env.sh +if [[ -f ${KYUUBI_ENV_SH} ]]; then + set -a + echo "Using kyuubi.sh environment file ${KYUUBI_ENV_SH} to initialize..." + . ${KYUUBI_ENV_SH} + set +a +else + echo "Warn: Not find kyuubi.sh environment file ${KYUUBI_ENV_SH}, using default ones..." +fi + +export KYUUBI_LOG_DIR="${KYUUBI_LOG_DIR:-"${KYUUBI_HOME}/logs"}" +if [[ -e ${KYUUBI_LOG_DIR} ]]; then + mkdir -p ${KYUUBI_LOG_DIR} +fi + +export KYUUBI_PID_DIR="${KYUUBI_PID_DIR:-"${KYUUBI_HOME}/pid"}" +if [[ -e ${KYUUBI_LOG_DIR} ]]; then + mkdir -p ${KYUUBI_LOG_DIR} +fi + +if [[ -z ${JAVA_HOME} ]]; then + if [[ $(command -v java) ]]; then + export JAVA_HOME="$(dirname $(dirname $(which java)))" + fi +fi + +export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.11"}" + +# Print essential environment variables to console +echo "JAVA_HOME: ${JAVA_HOME}" + +echo "KYUUBI_HOME: ${KYUUBI_HOME}" +echo "KYUUBI_CONF_DIR: ${KYUUBI_CONF_DIR}" +echo "KYUUBI_LOG_DIR: ${KYUUBI_LOG_DIR}" +echo "KYUUBI_PID_DIR: ${KYUUBI_PID_DIR}" + +echo "SPARK_HOME: ${SPARK_HOME}" +echo "SPARK_CONF_DIR: ${SPARK_CONF_DIR}" + +echo "HADOOP_CONF_DIR: ${HADOOP_CONF_DIR}" diff --git a/conf/kyuubi-env.sh.template b/conf/kyuubi-env.sh.template new file mode 100755 index 000000000..9c4a1b38a --- /dev/null +++ b/conf/kyuubi-env.sh.template @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# - JAVA_HOME Java runtime to use. By default use "java" from PATH. +# +# +# - KYUUBI_CONF_DIR Directory containing the Kyuubi configurations to use. +# (Default: $KYUUBI_HOME/conf) +# - KYUUBI_LOG_DIR Directory for Kyuubi server-side logs. +# (Default: $KYUUBI_HOME/logs) +# - KYUUBI_PID_DIR Directory stores the Kyuubi instance pid file. +# (Default: $KYUUBI_HOME/pid) +# - KYUUBI_MAX_LOG_FILES Maximum number of Kyuubi server logs can rotate to. +# (Default: 5) +# - KYUUBI_JAVA_OPTS JVM options for the Kyuubi server itself in the form "-Dx=y". +# (Default: none). +# - KYUUBI_NICENESS The scheduling priority for Kyuubi server. +# (Default: 0) +# +# - HADOOP_CONF_DIR Directory containing the Hadoop / YARN configuration to use. +# +# - SPARK_HOME Spark distribution which you would like to use in Kyuubi. +# - SPARK_CONF_DIR Optional directory where the Spark configuration lives. +# (Default: $SPARK_HOME/conf) diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml new file mode 100644 index 000000000..9dba244e6 --- /dev/null +++ b/kyuubi-assembly/pom.xml @@ -0,0 +1,49 @@ + + + + + 4.0.0 + + + kyuubi + yaooqinn + 0.8.0-SNAPSHOT + ../pom.xml + + + kyuubi-assembly + pom + Kyuubi Project Assembly + + + + yaooqinn + kyuubi-common + ${project.version} + + + + yaooqinn + kyuubi-thrift + ${project.version} + + + + \ No newline at end of file diff --git a/kyuubi-common/pom.xml b/kyuubi-common/pom.xml index e90d21a75..a1a2e3ec0 100644 --- a/kyuubi-common/pom.xml +++ b/kyuubi-common/pom.xml @@ -38,8 +38,8 @@ - org.apache.hadoop - hadoop-client + org.slf4j + slf4j-api diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiConf.scala new file mode 100644 index 000000000..9be89c59f --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiConf.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi + +class KyuubiConf { + +} + +object KyuubiConf { + /** a custom directory that contains the [[KYUUBI_CONF_FILE_NAME]] */ + final val KYUUBI_CONF_DIR = "KYUUBI_CONF_DIR" + /** the default file that contains kyuubi properties */ + final val KYUUBI_CONF_FILE_NAME = "kyuubi-defaults.conf" + final val KYUUBI_HOME = "KYUUBI_HOME" +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiException.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiException.scala new file mode 100644 index 000000000..c1aeb8e15 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiException.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi + +class KyuubiException(message: String, cause: Throwable) extends Exception(message, cause) { + + def this(message: String) = this(message, null) + +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala new file mode 100644 index 000000000..bdc303b87 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi + +import java.io.{File, InputStreamReader, IOException} +import java.nio.charset.StandardCharsets +import java.util.Properties + +import scala.collection.JavaConverters._ + +import yaooqinn.kyuubi.Logging + +private[kyuubi] object Utils extends Logging { + + import KyuubiConf._ + + def getSystemProperties: Map[String, String] = { + System.getProperties.asScala.toMap + } + + def getDefaultPropertiesFile(env: Map[String, String] = sys.env): Option[File] = { + env.get(KYUUBI_CONF_DIR) + .orElse(env.get(KYUUBI_HOME).map(_ + File.separator + "/conf")) + .map( d => new File(d + File.separator + KYUUBI_CONF_FILE_NAME)) + .filter(f => f.exists() && f.isFile) + } + + def getPropertiesFromFile(file: Option[File]): Map[String, String] = { + file.map { f => + val reader = new InputStreamReader(f.toURI.toURL.openStream(), StandardCharsets.UTF_8) + try { + val properties = new Properties() + properties.load(reader) + properties.stringPropertyNames().asScala.map { k => + (k, properties.getProperty(k).trim) + }.toMap + } catch { + case e: IOException => + throw new KyuubiException( + s"Failed when loading Kyuubi properties from ${f.getAbsolutePath}", e) + } finally { + reader.close() + } + }.getOrElse(Map.empty) + } + +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala new file mode 100644 index 000000000..994c0d2a8 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.server + +object KyuubiServer { + + def main(args: Array[String]): Unit = { + Thread.sleep(10000) + print("Hello Kyuubi") + } +} diff --git a/kyuubi-server/src/main/scala/yaooqinn/kyuubi/operation/KyuubiOperation.scala b/kyuubi-server/src/main/scala/yaooqinn/kyuubi/operation/KyuubiOperation.scala index 5fa83358d..9ca999244 100644 --- a/kyuubi-server/src/main/scala/yaooqinn/kyuubi/operation/KyuubiOperation.scala +++ b/kyuubi-server/src/main/scala/yaooqinn/kyuubi/operation/KyuubiOperation.scala @@ -17,11 +17,9 @@ package yaooqinn.kyuubi.operation -import org.apache.hadoop.hive.ql.session.OperationLog import org.apache.hive.service.cli.thrift.TProtocolVersion import org.apache.spark.sql.types.StructType -import yaooqinn.kyuubi.KyuubiSQLException import yaooqinn.kyuubi.cli.FetchOrientation import yaooqinn.kyuubi.schema.RowSet import yaooqinn.kyuubi.session.KyuubiSession diff --git a/pom.xml b/pom.xml index 7b3e9dd04..dc02110f7 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,7 @@ kyuubi-common kyuubi-thrift kyuubi-server + kyuubi-assembly pom @@ -52,10 +53,10 @@ UTF-8 1.7 - 2.11.8 + 2.11.12 3.0.3 2.11 - 3.5.4 + 3.6.3 org.apache.spark 2.4.5 provided @@ -65,6 +66,7 @@ 1.2.1.spark2 provided 1.1 + ${project.build.directory}/scala-${scala.binary.version}/jars 2.0.0-M15 2.6.0 3.1.2 @@ -150,7 +152,7 @@ org.scala-lang scala-library ${scala.version} - provided + compile @@ -371,7 +373,7 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - + org.apache.maven.plugins @@ -498,8 +500,64 @@ 3.0.2 + + org.apache.maven.plugins + maven-assembly-plugin + 3.1.0 + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.1.1 + + + default-cli + + build-classpath + + + + runtime + + + + - + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + generate-test-classpath + test-compile + + build-classpath + + + test + test_classpath + + + + copy-module-dependencies + package + + copy-dependencies + + + runtime + ${jars.target.dir} + + + + +