apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

new scripts for new framework

Browse Source
This commit is contained in:
Kent Yao 2020-05-22 19:16:04 +08:00
parent 7d389c44b2
commit b9a50d62ee
11 changed files with 555 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

197
bin/kyuubi.sh Executable file
View File

@ -0,0 +1,197 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
## Kyuubi Server Main Entrance
CLASS="org.apache.kyuubi.server.KyuubiServer"
function usage {
echo "Usage: ./bin/kyuubi.sh (start|stop|status)"
}
if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
usage
exit 0
fi
function kyuubi_rotate_log() {
log=$1;
if [[ -z ${KYUUBI_MAX_LOG_FILES} ]]; then
num=5
elif [[ ${KYUUBI_MAX_LOG_FILES} -gt 0 ]]; then
num=${KYUUBI_MAX_LOG_FILES}
else
echo "Error: KYUUBI_MAX_LOG_FILES must be a positive number, but got ${KYUUBI_MAX_LOG_FILES}"
exit -1
fi
if [ -f "$log" ]; then # rotate logs
while [ ${num} -gt 1 ]; do
prev=`expr ${num} - 1`
[ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
num=${prev}
done
mv "$log" "$log.$num";
fi
}
export KYUUBI_HOME="$(cd "`dirname "$0"`"/..; pwd)"
echo "Starting Kyuubi Server from ${KYUUBI_HOME}"
. "${KYUUBI_HOME}/bin/load-kyuubi-env.sh"
if [[ -z ${JAVA_HOME} ]]; then
echo "Error: JAVA_HOME IS NOT SET! CANNOT PROCEED."
exit 1
fi
if [[ -z ${SPARK_HOME} ]]; then
echo "Error: SPARK_HOME IS NOT SET! CANNOT PROCEED." >&2
exit 1
else
if [[ ! -x "${SPARK_HOME}/bin/spark-submit" ]]; then
echo echo "Error: INVALID SPARK DISTRIBUTION! CANNOT PROCEED." >&2
exit 1
fi
fi
RUNNER="${JAVA_HOME}/bin/java"
COMMAND="${RUNNER} ${KYUUBI_JAVA_OPTS} -cp"
## Find the Kyuubi Jar
if [[ -z "$KYUUBI_JAR_DIR" ]]; then
KYUUBI_JAR_DIR="$KYUUBI_HOME/jars"
if [[ ! -d ${KYUUBI_JAR_DIR} ]]; then
echo -e "\nCandidate Kyuubi lib $KYUUBI_JAR_DIR doesn't exist, searching development environment..."
KYUUBI_JAR_DIR="$KYUUBI_HOME/kyuubi-assembly/target/scala-${KYUUBI_SCALA_VERSION}/jars"
fi
fi
pid="${KYUUBI_PID_DIR}/kyuubi-$USER-$CLASS.pid"
function start_kyuubi() {
if [[ ! -w ${KYUUBI_PID_DIR} ]]; then
echo "${USER} does not have 'w' permission to ${KYUUBI_PID_DIR}"
exit 1
fi
if [[ ! -w ${KYUUBI_LOG_DIR} ]]; then
echo "${USER} does not have 'w' permission to ${KYUUBI_LOG_DIR}"
exit 1
fi
if [ -f "$pid" ]; then
TARGET_ID="$(cat "$pid")"
if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then
echo "$CLASS running as process $TARGET_ID Stop it first."
exit 1
fi
fi
log="${KYUUBI_LOG_DIR}/kyuubi-$USER-$CLASS-$HOSTNAME.out"
kyuubi_rotate_log ${log}
KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}"
cmd="${RUNNER} ${KYUUBI_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS"
echo "Starting $CLASS, logging to $log"
nohup nice -n "${KYUUBI_NICENESS:-0}" ${cmd} >> ${log} 2>&1 < /dev/null &
newpid="$!"
echo "$newpid" > "$pid"
# Poll for up to 5 seconds for the java process to start
for i in {1..10}
do
if [[ $(ps -p "$newpid" -o comm=) =~ "java" ]]; then
break
fi
sleep 0.5
done
sleep 2
# Check if the process has died; in that case we'll tail the log so the user can see
if [[ ! $(ps -p "$newpid" -o comm=) =~ "java" ]]; then
echo "Failed to launch: ${cmd}"
tail -2 "$log" | sed 's/^/ /'
echo "Full log in $log"
else
echo "Welcome to"
cat ${KYUUBI_HOME}/bin/kyuubi-logo
fi
}
function stop_kyuubi() {
if [ -f ${pid} ]; then
TARGET_ID="$(cat "$pid")"
if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then
echo "Stopping $CLASS"
kill "$TARGET_ID" && rm -f "$pid"
for i in {1..20}
do
sleep 0.5
if [[ ! $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then
break
fi
done
if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then
echo "Failed to stop kyuubi after 10 seconds, try 'kill -9 ${TARGET_ID}' forcefully "
else
cat ${KYUUBI_HOME}/bin/kyuubi-logo
echo "Bye!"
fi
else
echo "no $CLASS to stop"
fi
else
echo "no $CLASS to stop"
fi
}
function check_kyuubi() {
if [[ -f ${pid} ]]; then
TARGET_ID="$(cat "$pid")"
if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then
echo "Kyuubi is running (pid: $TARGET_ID)"
else
echo "Kyuubi is not running"
fi
else
echo "Kyuubi is not running"
fi
}
case $1 in
(start | "")
start_kyuubi
;;
(stop)
stop_kyuubi
;;
(status)
check_kyuubi
;;
(*)
usage
;;
esac

63
bin/load-kyuubi-env.sh Executable file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
export KYUUBI_HOME="${KYUUBI_HOME:-"$(cd "`dirname $0`"/..; pwd)"}"
export KYUUBI_CONF_DIR="${KYUUBI_CONF_DIR:-"${KYUUBI_HOME}"/conf}"
KYUUBI_ENV_SH="${KYUUBI_CONF_DIR}"/kyuubi-env.sh
if [[ -f ${KYUUBI_ENV_SH} ]]; then
set -a
echo "Using kyuubi.sh environment file ${KYUUBI_ENV_SH} to initialize..."
. ${KYUUBI_ENV_SH}
set +a
else
echo "Warn: Not find kyuubi.sh environment file ${KYUUBI_ENV_SH}, using default ones..."
fi
export KYUUBI_LOG_DIR="${KYUUBI_LOG_DIR:-"${KYUUBI_HOME}/logs"}"
if [[ -e ${KYUUBI_LOG_DIR} ]]; then
mkdir -p ${KYUUBI_LOG_DIR}
fi
export KYUUBI_PID_DIR="${KYUUBI_PID_DIR:-"${KYUUBI_HOME}/pid"}"
if [[ -e ${KYUUBI_LOG_DIR} ]]; then
mkdir -p ${KYUUBI_LOG_DIR}
fi
if [[ -z ${JAVA_HOME} ]]; then
if [[ $(command -v java) ]]; then
export JAVA_HOME="$(dirname $(dirname $(which java)))"
fi
fi
export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.11"}"
# Print essential environment variables to console
echo "JAVA_HOME: ${JAVA_HOME}"
echo "KYUUBI_HOME: ${KYUUBI_HOME}"
echo "KYUUBI_CONF_DIR: ${KYUUBI_CONF_DIR}"
echo "KYUUBI_LOG_DIR: ${KYUUBI_LOG_DIR}"
echo "KYUUBI_PID_DIR: ${KYUUBI_PID_DIR}"
echo "SPARK_HOME: ${SPARK_HOME}"
echo "SPARK_CONF_DIR: ${SPARK_CONF_DIR}"
echo "HADOOP_CONF_DIR: ${HADOOP_CONF_DIR}"

39
conf/kyuubi-env.sh.template Executable file
View File

@ -0,0 +1,39 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# - JAVA_HOME Java runtime to use. By default use "java" from PATH.
#
#
# - KYUUBI_CONF_DIR Directory containing the Kyuubi configurations to use.
# (Default: $KYUUBI_HOME/conf)
# - KYUUBI_LOG_DIR Directory for Kyuubi server-side logs.
# (Default: $KYUUBI_HOME/logs)
# - KYUUBI_PID_DIR Directory stores the Kyuubi instance pid file.
# (Default: $KYUUBI_HOME/pid)
# - KYUUBI_MAX_LOG_FILES Maximum number of Kyuubi server logs can rotate to.
# (Default: 5)
# - KYUUBI_JAVA_OPTS JVM options for the Kyuubi server itself in the form "-Dx=y".
# (Default: none).
# - KYUUBI_NICENESS The scheduling priority for Kyuubi server.
# (Default: 0)
#
# - HADOOP_CONF_DIR Directory containing the Hadoop / YARN configuration to use.
#
# - SPARK_HOME Spark distribution which you would like to use in Kyuubi.
# - SPARK_CONF_DIR Optional directory where the Spark configuration lives.
# (Default: $SPARK_HOME/conf)

49
kyuubi-assembly/pom.xml Normal file
View File

@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>kyuubi</artifactId>
<groupId>yaooqinn</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>kyuubi-assembly</artifactId>
<packaging>pom</packaging>
<name>Kyuubi Project Assembly</name>
<dependencies>
<dependency>
<groupId>yaooqinn</groupId>
<artifactId>kyuubi-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>yaooqinn</groupId>
<artifactId>kyuubi-thrift</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

4
kyuubi-common/pom.xml
View File

@ -38,8 +38,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
</dependencies>

30
kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiConf.scala Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi
class KyuubiConf {
}
object KyuubiConf {
/** a custom directory that contains the [[KYUUBI_CONF_FILE_NAME]] */
final val KYUUBI_CONF_DIR = "KYUUBI_CONF_DIR"
/** the default file that contains kyuubi properties */
final val KYUUBI_CONF_FILE_NAME = "kyuubi-defaults.conf"
final val KYUUBI_HOME = "KYUUBI_HOME"
}

24
kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiException.scala Normal file
View File

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi
class KyuubiException(message: String, cause: Throwable) extends Exception(message, cause) {
def this(message: String) = this(message, null)
}

62
kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala Normal file
View File

@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi
import java.io.{File, InputStreamReader, IOException}
import java.nio.charset.StandardCharsets
import java.util.Properties
import scala.collection.JavaConverters._
import yaooqinn.kyuubi.Logging
private[kyuubi] object Utils extends Logging {
import KyuubiConf._
def getSystemProperties: Map[String, String] = {
System.getProperties.asScala.toMap
}
def getDefaultPropertiesFile(env: Map[String, String] = sys.env): Option[File] = {
env.get(KYUUBI_CONF_DIR)
.orElse(env.get(KYUUBI_HOME).map(_ + File.separator + "/conf"))
.map( d => new File(d + File.separator + KYUUBI_CONF_FILE_NAME))
.filter(f => f.exists() && f.isFile)
}
def getPropertiesFromFile(file: Option[File]): Map[String, String] = {
file.map { f =>
val reader = new InputStreamReader(f.toURI.toURL.openStream(), StandardCharsets.UTF_8)
try {
val properties = new Properties()
properties.load(reader)
properties.stringPropertyNames().asScala.map { k =>
(k, properties.getProperty(k).trim)
}.toMap
} catch {
case e: IOException =>
throw new KyuubiException(
s"Failed when loading Kyuubi properties from ${f.getAbsolutePath}", e)
} finally {
reader.close()
}
}.getOrElse(Map.empty)
}
}

26
kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala Normal file
View File

@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.server
object KyuubiServer {
def main(args: Array[String]): Unit = {
Thread.sleep(10000)
print("Hello Kyuubi")
}
}

2
kyuubi-server/src/main/scala/yaooqinn/kyuubi/operation/KyuubiOperation.scala
View File

@ -17,11 +17,9 @@
package yaooqinn.kyuubi.operation
import org.apache.hadoop.hive.ql.session.OperationLog
import org.apache.hive.service.cli.thrift.TProtocolVersion
import org.apache.spark.sql.types.StructType
import yaooqinn.kyuubi.KyuubiSQLException
import yaooqinn.kyuubi.cli.FetchOrientation
import yaooqinn.kyuubi.schema.RowSet
import yaooqinn.kyuubi.session.KyuubiSession

68
pom.xml
View File

@ -28,6 +28,7 @@
<module>kyuubi-common</module>
<module>kyuubi-thrift</module>
<module>kyuubi-server</module>
<module>kyuubi-assembly</module>
</modules>
<packaging>pom</packaging>
@ -52,10 +53,10 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>1.7</java.version>
<scala.version>2.11.8</scala.version>
<scala.version>2.11.12</scala.version>
<scalatest.version>3.0.3</scalatest.version>
<scala.binary.version>2.11</scala.binary.version>
<maven.version>3.5.4</maven.version>
<maven.version>3.6.3</maven.version>
<spark.group>org.apache.spark</spark.group>
<spark.version>2.4.5</spark.version>
<spark.scope>provided</spark.scope>
@ -65,6 +66,7 @@
<hive.version>1.2.1.spark2</hive.version>
<hive.deps.scope>provided</hive.deps.scope>
<jpam.version>1.1</jpam.version>
<jars.target.dir>${project.build.directory}/scala-${scala.binary.version}/jars</jars.target.dir>
<apacheds.version>2.0.0-M15</apacheds.version>
<curator.version>2.6.0</curator.version>
<codahale.metrics.version>3.1.2</codahale.metrics.version>
@ -150,7 +152,7 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
<scope>compile</scope>
</dependency>
<dependency>
@ -371,7 +373,7 @@
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
<pluginManagement>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@ -498,8 +500,64 @@
<version>3.0.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.1.1</version>
<executions>
<execution>
<id>default-cli</id>
<goals>
<goal>build-classpath</goal>
</goals>
<configuration>
<!-- This includes dependencies with 'runtime' and 'compile' scopes;
see the docs for includeScope for more details -->
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
</pluginManagement>
<plugins>
<!-- This plugin dumps the test classpath into a file -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>generate-test-classpath</id>
<phase>test-compile</phase>
<goals>
<goal>build-classpath</goal>
</goals>
<configuration>
<includeScope>test</includeScope>
<outputProperty>test_classpath</outputProperty>
</configuration>
</execution>
<execution>
<id>copy-module-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeScope>runtime</includeScope>
<outputDirectory>${jars.target.dir}</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>