diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml
index 94eda6713..519c73237 100644
--- a/kyuubi-server/pom.xml
+++ b/kyuubi-server/pom.xml
@@ -126,12 +126,6 @@
true
-
-
- ${project.build.directory}/extra-resources
- true
-
-
org.apache.maven.plugins
@@ -155,6 +149,23 @@
run
+
+ build-info-test
+ generate-test-resources
+
+
+
+
+
+
+
+
+
+
+
+ run
+
+
fake-jar-test
generate-test-resources
@@ -243,18 +254,6 @@
build-helper-maven-plugin
3.0.0
-
- add-default-sources
- generate-sources
-
- add-source
-
-
-
- v${hive.version.short}/src/main/scala
-
-
-
add-source
generate-sources
@@ -288,14 +287,9 @@
${project.build.directory}/surefire-reports
.
TestSuite.txt
- -Xmx3g -Xss4m -XX:ReservedCodeCacheSize=512m
-
${project.build.testOutputDirectory}/kyuubi-server-${project.version}.jar
-
- ${project.build.testOutputDirectory}/tmp
-
diff --git a/kyuubi-server/src/main/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClient.scala b/kyuubi-server/src/main/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClient.scala
index 7506c9cdb..d8a0533bf 100644
--- a/kyuubi-server/src/main/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClient.scala
+++ b/kyuubi-server/src/main/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClient.scala
@@ -22,6 +22,7 @@ import java.net.URI
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.security.PrivilegedExceptionAction
+import java.text.SimpleDateFormat
import java.util.{Locale, Properties, UUID}
import java.util.concurrent.Callable
import java.util.zip.{ZipEntry, ZipOutputStream}
@@ -129,11 +130,11 @@ private[yarn] class KyuubiYarnClient(conf: SparkConf) extends Logging {
case ioe: IOException =>
warn("Failed to cleanup staging dir " + appStagingDir, ioe)
} finally {
- killApplication
+ killApplication()
}
}
- def killApplication: Unit = {
+ def killApplication(): Unit = {
try {
yarnClient.killApplication(appId)
} catch {
@@ -290,8 +291,8 @@ private[yarn] class KyuubiYarnClient(conf: SparkConf) extends Logging {
}
val localResources = new HashMap[String, LocalResource]
FileSystem.mkdirs(fs, appStagingDir, new FsPermission(STAGING_DIR_PERMISSION))
- val symlinkCache: Map[URI, Path] = HashMap[URI, Path]()
- val statCache: Map[URI, FileStatus] = HashMap[URI, FileStatus]()
+ val symlinkCache = HashMap[URI, Path]()
+ val statCache = HashMap[URI, FileStatus]()
/**
* Copy the non-local file to HDFS (if not already there) and add it to the Application Master's
@@ -470,15 +471,17 @@ private[yarn] class KyuubiYarnClient(conf: SparkConf) extends Logging {
}
private[this] val stateChecker: Callable[Boolean] = new Callable[Boolean] {
- private val timeout = conf.getTimeAsMs(KyuubiConf.YARN_CONTAINER_TIMEOUT.key)
+ private val timeout = conf.getTimeAsMs(KyuubiConf.YARN_CONTAINER_TIMEOUT.key, "60s")
import YarnApplicationState._
override def call(): Boolean = {
try {
val report = yarnClient.getApplicationReport(appId)
- info(formatReportDetails(report))
- report.getYarnApplicationState match {
+ val state = report.getYarnApplicationState
+ if (state != ACCEPTED) info(formatReportDetails(report))
+ info(s"Application report for $appId[$state]")
+ state match {
case RUNNING => false
case ACCEPTED | NEW | NEW_SAVING | SUBMITTED =>
if (System.currentTimeMillis - report.getStartTime < timeout) {
@@ -502,18 +505,17 @@ private[yarn] class KyuubiYarnClient(conf: SparkConf) extends Logging {
}
def formatReportDetails(report: ApplicationReport): String = {
- val details = Seq[(String, String)](
+ Seq(
("Client token", Option(report.getClientToAMToken).map(_.toString).getOrElse("")),
("Diagnostics", report.getDiagnostics),
("Kyuubi server host", report.getHost),
("Bind port", report.getRpcPort.toString),
("Queue", report.getQueue),
- ("Start time", report.getStartTime.toString),
+ ("Start time", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(report.getStartTime)),
("Final status", Option(report.getFinalApplicationStatus).map(_.toString).getOrElse("")),
("Tracking URL", report.getTrackingUrl),
("User", report.getUser)
- )
- details.map { case (k, v) =>
+ ).map { case (k, v) =>
val newValue = Option(v).filter(_.nonEmpty).getOrElse("N/A")
s"\n\t $k: $newValue"
}.mkString("")
@@ -619,7 +621,6 @@ object KyuubiYarnClient {
*/
def startKyuubiAppMaster(): Unit = {
val conf = new SparkConf()
- KyuubiConf.getAllDefaults.foreach(kv => conf.setIfMissing(kv._1, kv._2))
new KyuubiYarnClient(conf).submit()
}
}
diff --git a/kyuubi-server/src/test/scala/yaooqinn/kyuubi/packageSuite.scala b/kyuubi-server/src/test/scala/yaooqinn/kyuubi/packageSuite.scala
index 3a157a654..bb393ea0d 100644
--- a/kyuubi-server/src/test/scala/yaooqinn/kyuubi/packageSuite.scala
+++ b/kyuubi-server/src/test/scala/yaooqinn/kyuubi/packageSuite.scala
@@ -36,7 +36,6 @@ class packageSuite extends SparkFunSuite {
assert(props.getProperty("revision") === REVISION)
assert(props.getProperty("user") === BUILD_USER)
assert(props.getProperty("url") === REPO_URL)
- assert(props.getProperty("date") === BUILD_DATE)
}
}
diff --git a/kyuubi-server/src/test/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClientSuite.scala b/kyuubi-server/src/test/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClientSuite.scala
index 6dcc1052e..4001416f6 100644
--- a/kyuubi-server/src/test/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClientSuite.scala
+++ b/kyuubi-server/src/test/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClientSuite.scala
@@ -131,8 +131,6 @@ class KyuubiYarnClientSuite extends SparkFunSuite with Matchers with MockitoSuga
test("kyuubi yarn client init") {
val conf = new SparkConf()
- KyuubiConf.getAllDefaults.foreach(kv => conf.setIfMissing(kv._1, kv._2))
-
val client = new KyuubiYarnClient(conf)
assert(ReflectUtils.getFieldValue(client,
"yaooqinn$kyuubi$yarn$KyuubiYarnClient$$hadoopConf").isInstanceOf[YarnConfiguration])
@@ -242,7 +240,6 @@ class KyuubiYarnClientSuite extends SparkFunSuite with Matchers with MockitoSuga
def withClientBase(f: (YarnClient, KyuubiYarnClient) => Any): Unit = {
val conf = new SparkConf()
- KyuubiConf.getAllDefaults.foreach(kv => conf.setIfMissing(kv._1, kv._2))
val client = new KyuubiYarnClient(conf)
val yarnClient = mock[YarnClient]
ReflectUtils.setFieldValue(client,
diff --git a/pom.xml b/pom.xml
index eee52bc8f..f928f5d0c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,7 +61,6 @@
provided
org.spark-project.hive
1.2.1.spark2
- 1.2.1
provided
1.1
2.0.0-M15