diff --git a/.gitignore b/.gitignore index 71be914a0..9be664088 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ kyuubi-server/kyuubi-kdc/ metrics/report.json metrics/.report.json.crc /kyuubi-ha/embedded_zookeeper/ +embedded_zookeeper/ diff --git a/bin/load-kyuubi-env.sh b/bin/load-kyuubi-env.sh index 339b51217..25dbad0b4 100755 --- a/bin/load-kyuubi-env.sh +++ b/bin/load-kyuubi-env.sh @@ -47,7 +47,7 @@ if [[ -z ${JAVA_HOME} ]]; then fi fi -export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.11"}" +export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.12"}" # Print essential environment variables to console echo "JAVA_HOME: ${JAVA_HOME}" diff --git a/conf/kyuubi-defaults.conf.template b/conf/kyuubi-defaults.conf.template new file mode 100644 index 000000000..e69de29bb diff --git a/conf/log4j.properties.template b/conf/log4j.properties.template new file mode 100644 index 000000000..e69de29bb diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml index cc84848a2..92cdc4efa 100644 --- a/kyuubi-assembly/pom.xml +++ b/kyuubi-assembly/pom.xml @@ -44,6 +44,88 @@ kyuubi-hive-thrift ${project.version} + + + org.apache.kyuubi + kyuubi-ha + ${project.version} + + + + org.apache.kyuubi + kyuubi-main + ${project.version} + + + + org.apache.curator + curator-test + compile + + + + org.apache.curator + curator-framework + compile + + + + com.google.guava + guava + compile + + + + org.apache.hadoop + hadoop-common + compile + + + com.sun.jersey + * + + + com.google.code.gson + gson + + + org.apache.commons + * + + + commons-cli + commons-cli + + + org.mortbay.jetty + * + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + log4j + log4j + + + commons-logging + commons-logging + + + org.htrace + htrace-core + + + org.apache.avro + avro + + + \ No newline at end of file diff --git a/kyuubi-common/src/main/resources/log4j-defaults.properties b/kyuubi-common/src/main/resources/log4j-defaults.properties new file mode 100644 index 000000000..426adcf6d --- /dev/null +++ b/kyuubi-common/src/main/resources/log4j-defaults.properties @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the console +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n \ No newline at end of file diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala index b05e4e867..21b5afca9 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala @@ -17,10 +17,37 @@ package org.apache.kyuubi -import org.slf4j.LoggerFactory +import org.apache.log4j.{Level, LogManager, PropertyConfigurator} +import org.slf4j.{Logger, LoggerFactory} +import org.slf4j.impl.StaticLoggerBinder +/** + * Simple version of logging adopted from Apache Spark. + */ trait Logging { - lazy val logger = LoggerFactory.getLogger(this.getClass) + + @transient private var log_ : Logger = _ + + // Method to get the logger name for this object + protected def loggerName: String = { + // Ignore trailing $'s in the class names for Scala objects + this.getClass.getName.stripSuffix("$") + } + + // Method to get or create the logger for this object + protected def logger: Logger = { + if (log_ == null) { + if (!Logging.initialized) { + Logging.initLock.synchronized { + if (!Logging.initialized) { + initializeLogging() + } + } + } + log_ = LoggerFactory.getLogger(loggerName) + } + log_ + } def debug(message: => Any): Unit = { if (logger.isDebugEnabled) { @@ -57,4 +84,46 @@ trait Logging { logger.error(message.toString) } } + + private def initializeLogging(): Unit = { + if (Logging.isLog4j12) { + val log4j12Initialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements + // scalastyle:off println + if (!log4j12Initialized) { + Logging.useDefault = true + val defaultLogProps = "log4j-defaults.properties" + Option(Thread.currentThread().getContextClassLoader.getResource(defaultLogProps)) match { + case Some(url) => + PropertyConfigurator.configure(url) + case None => + System.err.println(s"Missing $defaultLogProps") + } + } + + val rootLogger = LogManager.getRootLogger + if (Logging.defaultRootLevel == null) { + Logging.defaultRootLevel = rootLogger.getLevel + } + // scalastyle:on println + } + Logging.initialized = true + + // Force a call into slf4j to initialize it. Avoids this happening from multiple threads + // and triggering this: http://mailman.qos.ch/pipermail/slf4j-dev/2010-April/002956.html + logger + } +} + +object Logging { + @volatile private var useDefault = false + @volatile private var defaultRootLevel: Level = _ + @volatile private var initialized = false + val initLock = new Object() + private def isLog4j12: Boolean = { + // This distinguishes the log4j 1.2 binding, currently + // org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently + // org.apache.logging.slf4j.Log4jLoggerFactory + val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr + "org.slf4j.impl.Log4jLoggerFactory".equals(binderClass) + } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala index 9c8999126..94ac07ef8 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala @@ -47,6 +47,7 @@ private[kyuubi] object Utils extends Logging { def getPropertiesFromFile(file: Option[File]): Map[String, String] = { file.map { f => + info(s"Loading Kyuubi properties from ${f.getAbsolutePath}") val reader = new InputStreamReader(f.toURI.toURL.openStream(), StandardCharsets.UTF_8) try { val properties = new Properties() diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 8d8b0e3d0..6f1758465 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -19,23 +19,29 @@ package org.apache.kyuubi.config import java.util.concurrent.ConcurrentHashMap -import org.apache.kyuubi.Utils +import org.apache.kyuubi.{Logging, Utils} -case class KyuubiConf(loadSysDefault: Boolean = true) { +case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { private val settings = new ConcurrentHashMap[String, String]() private lazy val reader: ConfigProvider = new ConfigProvider(settings) if (loadSysDefault) { - loadSysProps() + loadFromMap() } - private def loadSysProps(): KyuubiConf = { - for ((key, value) <- Utils.getSystemProperties if key.startsWith(KYUUBI_PREFIX)) { + private def loadFromMap(props: Map[String, String] = Utils.getSystemProperties): KyuubiConf = { + for ((key, value) <- props if key.startsWith(KYUUBI_PREFIX)) { set(key, value) } this } + def loadFileDefaults(): KyuubiConf = { + val maybeConfigFile = Utils.getDefaultPropertiesFile() + loadFromMap(Utils.getPropertiesFromFile(maybeConfigFile)) + this + } + def set[T](entry: ConfigEntry[T], value: T): KyuubiConf = { settings.put(entry.key, entry.strConverter(value)) this diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/server/EmbeddedZkServer.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/server/EmbeddedZkServer.scala index 5a39bba17..6f37f8fcb 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/server/EmbeddedZkServer.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/server/EmbeddedZkServer.scala @@ -21,6 +21,7 @@ import java.io.File import org.apache.curator.test.TestingServer +import org.apache.kyuubi.Logging import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.AbstractService @@ -32,7 +33,7 @@ import org.apache.kyuubi.service.AbstractService * * @param name the service name */ -class EmbeddedZkServer private(name: String) extends AbstractService(name) { +class EmbeddedZkServer private(name: String) extends AbstractService(name) with Logging { def this() = this(classOf[EmbeddedZkServer].getSimpleName) @@ -51,6 +52,7 @@ class EmbeddedZkServer private(name: String) extends AbstractService(name) { override def start(): Unit = { server.start() + info(s"$getName is started at $getConnectString") super.start() } diff --git a/kyuubi-main/pom.xml b/kyuubi-main/pom.xml new file mode 100644 index 000000000..9d644798e --- /dev/null +++ b/kyuubi-main/pom.xml @@ -0,0 +1,46 @@ + + + + + + kyuubi + org.apache.kyuubi + 1.0.0-SNAPSHOT + + 4.0.0 + + kyuubi-main + Kyuubi Project Main Service + jar + + + + org.apache.kyuubi + kyuubi-common + ${project.version} + + + org.apache.kyuubi + kyuubi-ha + ${project.version} + + + + \ No newline at end of file diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala b/kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala similarity index 58% rename from kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala rename to kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala index 994c0d2a8..aea78ec27 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala +++ b/kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala @@ -17,10 +17,40 @@ package org.apache.kyuubi.server +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.ha.server.EmbeddedZkServer +import org.apache.kyuubi.service.CompositeService + object KyuubiServer { def main(args: Array[String]): Unit = { + val conf = new KyuubiConf().loadFileDefaults() + val server = new KyuubiServer() + server.initialize(conf) + server.start() + Thread.sleep(10000) print("Hello Kyuubi") } } + +class KyuubiServer(name: String) extends CompositeService(name) { + + def this() = this(classOf[KyuubiServer].getName) + + override def initialize(conf: KyuubiConf): Unit = { + this.conf = conf + val zkServer = new EmbeddedZkServer() + addService(zkServer) + super.initialize(conf) + } + + override def start(): Unit = { + super.start() + } + + override def stop(): Unit = { + super.stop() + } + +} diff --git a/kyuubi-spark-sql-engine/pom.xml b/kyuubi-spark-sql-engine/pom.xml index aa5c36d10..13878cb26 100644 --- a/kyuubi-spark-sql-engine/pom.xml +++ b/kyuubi-spark-sql-engine/pom.xml @@ -64,6 +64,29 @@ net.alchim31.maven scala-maven-plugin + + + org.apache.maven.plugins + maven-shade-plugin + + false + + + org.apache.kyuubi:kyuubi-common + org.apache.kyuubi:kyuubi-ha + + + + + + package + + shade + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 605c261dd..e90abce01 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,7 @@ kyuubi-hive-thrift kyuubi-spark-sql-engine kyuubi-ha + kyuubi-main pom @@ -282,8 +283,8 @@ org.slf4j slf4j-api - 1.7.16 - provided + 1.7.30 + compile