diff --git a/.gitignore b/.gitignore
index 71be914a0..9be664088 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,3 +43,4 @@ kyuubi-server/kyuubi-kdc/
metrics/report.json
metrics/.report.json.crc
/kyuubi-ha/embedded_zookeeper/
+embedded_zookeeper/
diff --git a/bin/load-kyuubi-env.sh b/bin/load-kyuubi-env.sh
index 339b51217..25dbad0b4 100755
--- a/bin/load-kyuubi-env.sh
+++ b/bin/load-kyuubi-env.sh
@@ -47,7 +47,7 @@ if [[ -z ${JAVA_HOME} ]]; then
fi
fi
-export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.11"}"
+export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.12"}"
# Print essential environment variables to console
echo "JAVA_HOME: ${JAVA_HOME}"
diff --git a/conf/kyuubi-defaults.conf.template b/conf/kyuubi-defaults.conf.template
new file mode 100644
index 000000000..e69de29bb
diff --git a/conf/log4j.properties.template b/conf/log4j.properties.template
new file mode 100644
index 000000000..e69de29bb
diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml
index cc84848a2..92cdc4efa 100644
--- a/kyuubi-assembly/pom.xml
+++ b/kyuubi-assembly/pom.xml
@@ -44,6 +44,88 @@
kyuubi-hive-thrift
${project.version}
+
+
+ org.apache.kyuubi
+ kyuubi-ha
+ ${project.version}
+
+
+
+ org.apache.kyuubi
+ kyuubi-main
+ ${project.version}
+
+
+
+ org.apache.curator
+ curator-test
+ compile
+
+
+
+ org.apache.curator
+ curator-framework
+ compile
+
+
+
+ com.google.guava
+ guava
+ compile
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ compile
+
+
+ com.sun.jersey
+ *
+
+
+ com.google.code.gson
+ gson
+
+
+ org.apache.commons
+ *
+
+
+ commons-cli
+ commons-cli
+
+
+ org.mortbay.jetty
+ *
+
+
+ javax.servlet
+ servlet-api
+
+
+ javax.servlet.jsp
+ jsp-api
+
+
+ log4j
+ log4j
+
+
+ commons-logging
+ commons-logging
+
+
+ org.htrace
+ htrace-core
+
+
+ org.apache.avro
+ avro
+
+
+
\ No newline at end of file
diff --git a/kyuubi-common/src/main/resources/log4j-defaults.properties b/kyuubi-common/src/main/resources/log4j-defaults.properties
new file mode 100644
index 000000000..426adcf6d
--- /dev/null
+++ b/kyuubi-common/src/main/resources/log4j-defaults.properties
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
\ No newline at end of file
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala
index b05e4e867..21b5afca9 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala
@@ -17,10 +17,37 @@
package org.apache.kyuubi
-import org.slf4j.LoggerFactory
+import org.apache.log4j.{Level, LogManager, PropertyConfigurator}
+import org.slf4j.{Logger, LoggerFactory}
+import org.slf4j.impl.StaticLoggerBinder
+/**
+ * Simple version of logging adopted from Apache Spark.
+ */
trait Logging {
- lazy val logger = LoggerFactory.getLogger(this.getClass)
+
+ @transient private var log_ : Logger = _
+
+ // Method to get the logger name for this object
+ protected def loggerName: String = {
+ // Ignore trailing $'s in the class names for Scala objects
+ this.getClass.getName.stripSuffix("$")
+ }
+
+ // Method to get or create the logger for this object
+ protected def logger: Logger = {
+ if (log_ == null) {
+ if (!Logging.initialized) {
+ Logging.initLock.synchronized {
+ if (!Logging.initialized) {
+ initializeLogging()
+ }
+ }
+ }
+ log_ = LoggerFactory.getLogger(loggerName)
+ }
+ log_
+ }
def debug(message: => Any): Unit = {
if (logger.isDebugEnabled) {
@@ -57,4 +84,46 @@ trait Logging {
logger.error(message.toString)
}
}
+
+ private def initializeLogging(): Unit = {
+ if (Logging.isLog4j12) {
+ val log4j12Initialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements
+ // scalastyle:off println
+ if (!log4j12Initialized) {
+ Logging.useDefault = true
+ val defaultLogProps = "log4j-defaults.properties"
+ Option(Thread.currentThread().getContextClassLoader.getResource(defaultLogProps)) match {
+ case Some(url) =>
+ PropertyConfigurator.configure(url)
+ case None =>
+ System.err.println(s"Missing $defaultLogProps")
+ }
+ }
+
+ val rootLogger = LogManager.getRootLogger
+ if (Logging.defaultRootLevel == null) {
+ Logging.defaultRootLevel = rootLogger.getLevel
+ }
+ // scalastyle:on println
+ }
+ Logging.initialized = true
+
+ // Force a call into slf4j to initialize it. Avoids this happening from multiple threads
+ // and triggering this: http://mailman.qos.ch/pipermail/slf4j-dev/2010-April/002956.html
+ logger
+ }
+}
+
+object Logging {
+ @volatile private var useDefault = false
+ @volatile private var defaultRootLevel: Level = _
+ @volatile private var initialized = false
+ val initLock = new Object()
+ private def isLog4j12: Boolean = {
+ // This distinguishes the log4j 1.2 binding, currently
+ // org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently
+ // org.apache.logging.slf4j.Log4jLoggerFactory
+ val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr
+ "org.slf4j.impl.Log4jLoggerFactory".equals(binderClass)
+ }
}
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
index 9c8999126..94ac07ef8 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
@@ -47,6 +47,7 @@ private[kyuubi] object Utils extends Logging {
def getPropertiesFromFile(file: Option[File]): Map[String, String] = {
file.map { f =>
+ info(s"Loading Kyuubi properties from ${f.getAbsolutePath}")
val reader = new InputStreamReader(f.toURI.toURL.openStream(), StandardCharsets.UTF_8)
try {
val properties = new Properties()
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 8d8b0e3d0..6f1758465 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -19,23 +19,29 @@ package org.apache.kyuubi.config
import java.util.concurrent.ConcurrentHashMap
-import org.apache.kyuubi.Utils
+import org.apache.kyuubi.{Logging, Utils}
-case class KyuubiConf(loadSysDefault: Boolean = true) {
+case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
private val settings = new ConcurrentHashMap[String, String]()
private lazy val reader: ConfigProvider = new ConfigProvider(settings)
if (loadSysDefault) {
- loadSysProps()
+ loadFromMap()
}
- private def loadSysProps(): KyuubiConf = {
- for ((key, value) <- Utils.getSystemProperties if key.startsWith(KYUUBI_PREFIX)) {
+ private def loadFromMap(props: Map[String, String] = Utils.getSystemProperties): KyuubiConf = {
+ for ((key, value) <- props if key.startsWith(KYUUBI_PREFIX)) {
set(key, value)
}
this
}
+ def loadFileDefaults(): KyuubiConf = {
+ val maybeConfigFile = Utils.getDefaultPropertiesFile()
+ loadFromMap(Utils.getPropertiesFromFile(maybeConfigFile))
+ this
+ }
+
def set[T](entry: ConfigEntry[T], value: T): KyuubiConf = {
settings.put(entry.key, entry.strConverter(value))
this
diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/server/EmbeddedZkServer.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/server/EmbeddedZkServer.scala
index 5a39bba17..6f37f8fcb 100644
--- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/server/EmbeddedZkServer.scala
+++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/server/EmbeddedZkServer.scala
@@ -21,6 +21,7 @@ import java.io.File
import org.apache.curator.test.TestingServer
+import org.apache.kyuubi.Logging
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.service.AbstractService
@@ -32,7 +33,7 @@ import org.apache.kyuubi.service.AbstractService
*
* @param name the service name
*/
-class EmbeddedZkServer private(name: String) extends AbstractService(name) {
+class EmbeddedZkServer private(name: String) extends AbstractService(name) with Logging {
def this() = this(classOf[EmbeddedZkServer].getSimpleName)
@@ -51,6 +52,7 @@ class EmbeddedZkServer private(name: String) extends AbstractService(name) {
override def start(): Unit = {
server.start()
+ info(s"$getName is started at $getConnectString")
super.start()
}
diff --git a/kyuubi-main/pom.xml b/kyuubi-main/pom.xml
new file mode 100644
index 000000000..9d644798e
--- /dev/null
+++ b/kyuubi-main/pom.xml
@@ -0,0 +1,46 @@
+
+
+
+
+
+ kyuubi
+ org.apache.kyuubi
+ 1.0.0-SNAPSHOT
+
+ 4.0.0
+
+ kyuubi-main
+ Kyuubi Project Main Service
+ jar
+
+
+
+ org.apache.kyuubi
+ kyuubi-common
+ ${project.version}
+
+
+ org.apache.kyuubi
+ kyuubi-ha
+ ${project.version}
+
+
+
+
\ No newline at end of file
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala b/kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
similarity index 58%
rename from kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
rename to kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
index 994c0d2a8..aea78ec27 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
+++ b/kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
@@ -17,10 +17,40 @@
package org.apache.kyuubi.server
+import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.ha.server.EmbeddedZkServer
+import org.apache.kyuubi.service.CompositeService
+
object KyuubiServer {
def main(args: Array[String]): Unit = {
+ val conf = new KyuubiConf().loadFileDefaults()
+ val server = new KyuubiServer()
+ server.initialize(conf)
+ server.start()
+
Thread.sleep(10000)
print("Hello Kyuubi")
}
}
+
+class KyuubiServer(name: String) extends CompositeService(name) {
+
+ def this() = this(classOf[KyuubiServer].getName)
+
+ override def initialize(conf: KyuubiConf): Unit = {
+ this.conf = conf
+ val zkServer = new EmbeddedZkServer()
+ addService(zkServer)
+ super.initialize(conf)
+ }
+
+ override def start(): Unit = {
+ super.start()
+ }
+
+ override def stop(): Unit = {
+ super.stop()
+ }
+
+}
diff --git a/kyuubi-spark-sql-engine/pom.xml b/kyuubi-spark-sql-engine/pom.xml
index aa5c36d10..13878cb26 100644
--- a/kyuubi-spark-sql-engine/pom.xml
+++ b/kyuubi-spark-sql-engine/pom.xml
@@ -64,6 +64,29 @@
net.alchim31.maven
scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+ false
+
+
+ org.apache.kyuubi:kyuubi-common
+ org.apache.kyuubi:kyuubi-ha
+
+
+
+
+
+ package
+
+ shade
+
+
+
+
+
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 605c261dd..e90abce01 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,6 +32,7 @@
kyuubi-hive-thrift
kyuubi-spark-sql-engine
kyuubi-ha
+ kyuubi-main
pom
@@ -282,8 +283,8 @@
org.slf4j
slf4j-api
- 1.7.16
- provided
+ 1.7.30
+ compile