apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Enable EmbeddedZkServer for KyuubiServer

Browse Source
This commit is contained in:
Kent Yao 2020-06-17 22:42:19 +08:00
parent e1d55f8351
commit 7fb0b1a32f
14 changed files with 295 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -43,3 +43,4 @@ kyuubi-server/kyuubi-kdc/
metrics/report.json
metrics/.report.json.crc
/kyuubi-ha/embedded_zookeeper/
embedded_zookeeper/

2
bin/load-kyuubi-env.sh
View File

@ -47,7 +47,7 @@ if [[ -z ${JAVA_HOME} ]]; then
fi
fi
export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.11"}"
export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.12"}"
# Print essential environment variables to console
echo "JAVA_HOME: ${JAVA_HOME}"

0
conf/kyuubi-defaults.conf.template Normal file
View File

0
conf/log4j.properties.template Normal file
View File

82
kyuubi-assembly/pom.xml
View File

@ -44,6 +44,88 @@
<artifactId>kyuubi-hive-thrift</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-ha</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-main</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>

23
kyuubi-common/src/main/resources/log4j-defaults.properties Normal file
View File

@ -0,0 +1,23 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Set everything to be logged to the console
log4j.rootCategory=INFO, console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n

73
kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala
View File

@ -17,10 +17,37 @@
package org.apache.kyuubi
import org.slf4j.LoggerFactory
import org.apache.log4j.{Level, LogManager, PropertyConfigurator}
import org.slf4j.{Logger, LoggerFactory}
import org.slf4j.impl.StaticLoggerBinder
/**
* Simple version of logging adopted from Apache Spark.
*/
trait Logging {
lazy val logger = LoggerFactory.getLogger(this.getClass)
@transient private var log_ : Logger = _
// Method to get the logger name for this object
protected def loggerName: String = {
// Ignore trailing $'s in the class names for Scala objects
this.getClass.getName.stripSuffix("$")
}
// Method to get or create the logger for this object
protected def logger: Logger = {
if (log_ == null) {
if (!Logging.initialized) {
Logging.initLock.synchronized {
if (!Logging.initialized) {
initializeLogging()
}
}
}
log_ = LoggerFactory.getLogger(loggerName)
}
log_
}
def debug(message: => Any): Unit = {
if (logger.isDebugEnabled) {
@ -57,4 +84,46 @@ trait Logging {
logger.error(message.toString)
}
}
private def initializeLogging(): Unit = {
if (Logging.isLog4j12) {
val log4j12Initialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements
// scalastyle:off println
if (!log4j12Initialized) {
Logging.useDefault = true
val defaultLogProps = "log4j-defaults.properties"
Option(Thread.currentThread().getContextClassLoader.getResource(defaultLogProps)) match {
case Some(url) =>
PropertyConfigurator.configure(url)
case None =>
System.err.println(s"Missing $defaultLogProps")
}
}
val rootLogger = LogManager.getRootLogger
if (Logging.defaultRootLevel == null) {
Logging.defaultRootLevel = rootLogger.getLevel
}
// scalastyle:on println
}
Logging.initialized = true
// Force a call into slf4j to initialize it. Avoids this happening from multiple threads
// and triggering this: http://mailman.qos.ch/pipermail/slf4j-dev/2010-April/002956.html
logger
}
}
object Logging {
@volatile private var useDefault = false
@volatile private var defaultRootLevel: Level = _
@volatile private var initialized = false
val initLock = new Object()
private def isLog4j12: Boolean = {
// This distinguishes the log4j 1.2 binding, currently
// org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently
// org.apache.logging.slf4j.Log4jLoggerFactory
val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr
"org.slf4j.impl.Log4jLoggerFactory".equals(binderClass)
}
}

1
kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
View File

@ -47,6 +47,7 @@ private[kyuubi] object Utils extends Logging {
def getPropertiesFromFile(file: Option[File]): Map[String, String] = {
file.map { f =>
info(s"Loading Kyuubi properties from ${f.getAbsolutePath}")
val reader = new InputStreamReader(f.toURI.toURL.openStream(), StandardCharsets.UTF_8)
try {
val properties = new Properties()

16
kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
View File

@ -19,23 +19,29 @@ package org.apache.kyuubi.config
import java.util.concurrent.ConcurrentHashMap
import org.apache.kyuubi.Utils
import org.apache.kyuubi.{Logging, Utils}
case class KyuubiConf(loadSysDefault: Boolean = true) {
case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
private val settings = new ConcurrentHashMap[String, String]()
private lazy val reader: ConfigProvider = new ConfigProvider(settings)
if (loadSysDefault) {
loadSysProps()
loadFromMap()
}
private def loadSysProps(): KyuubiConf = {
for ((key, value) <- Utils.getSystemProperties if key.startsWith(KYUUBI_PREFIX)) {
private def loadFromMap(props: Map[String, String] = Utils.getSystemProperties): KyuubiConf = {
for ((key, value) <- props if key.startsWith(KYUUBI_PREFIX)) {
set(key, value)
}
this
}
def loadFileDefaults(): KyuubiConf = {
val maybeConfigFile = Utils.getDefaultPropertiesFile()
loadFromMap(Utils.getPropertiesFromFile(maybeConfigFile))
this
}
def set[T](entry: ConfigEntry[T], value: T): KyuubiConf = {
settings.put(entry.key, entry.strConverter(value))
this

4
kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/server/EmbeddedZkServer.scala
View File

@ -21,6 +21,7 @@ import java.io.File
import org.apache.curator.test.TestingServer
import org.apache.kyuubi.Logging
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.service.AbstractService
@ -32,7 +33,7 @@ import org.apache.kyuubi.service.AbstractService
*
* @param name the service name
*/
class EmbeddedZkServer private(name: String) extends AbstractService(name) {
class EmbeddedZkServer private(name: String) extends AbstractService(name) with Logging {
def this() = this(classOf[EmbeddedZkServer].getSimpleName)
@ -51,6 +52,7 @@ class EmbeddedZkServer private(name: String) extends AbstractService(name) {
override def start(): Unit = {
server.start()
info(s"$getName is started at $getConnectString")
super.start()
}

46
kyuubi-main/pom.xml Normal file
View File

@ -0,0 +1,46 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>kyuubi</artifactId>
<groupId>org.apache.kyuubi</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>kyuubi-main</artifactId>
<name>Kyuubi Project Main Service</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-ha</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

30
kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala → kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
View File

@ -17,10 +17,40 @@
package org.apache.kyuubi.server
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ha.server.EmbeddedZkServer
import org.apache.kyuubi.service.CompositeService
object KyuubiServer {
def main(args: Array[String]): Unit = {
val conf = new KyuubiConf().loadFileDefaults()
val server = new KyuubiServer()
server.initialize(conf)
server.start()
Thread.sleep(10000)
print("Hello Kyuubi")
}
}
class KyuubiServer(name: String) extends CompositeService(name) {
def this() = this(classOf[KyuubiServer].getName)
override def initialize(conf: KyuubiConf): Unit = {
this.conf = conf
val zkServer = new EmbeddedZkServer()
addService(zkServer)
super.initialize(conf)
}
override def start(): Unit = {
super.start()
}
override def stop(): Unit = {
super.stop()
}
}

23
kyuubi-spark-sql-engine/pom.xml
View File

@ -64,6 +64,29 @@
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<configuration>
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
<include>org.apache.kyuubi:kyuubi-common</include>
<include>org.apache.kyuubi:kyuubi-ha</include>
</includes>
</artifactSet>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

5
pom.xml
View File

@ -32,6 +32,7 @@
<module>kyuubi-hive-thrift</module>
<module>kyuubi-spark-sql-engine</module>
<module>kyuubi-ha</module>
<module>kyuubi-main</module>
</modules>
<packaging>pom</packaging>
@ -282,8 +283,8 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.16</version>
<scope>provided</scope>
<version>1.7.30</version>
<scope>compile</scope>
</dependency>
<dependency>