From 3b98cdd06384bae4442dd8d31d9f17ef5bcb4090 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Fri, 27 Jul 2018 20:02:52 +0800 Subject: [PATCH] fix travis; fix port error --- kyuubi-server/pom.xml | 40 ++++++++----------- .../kyuubi/yarn/KyuubiYarnClient.scala | 25 ++++++------ .../scala/yaooqinn/kyuubi/packageSuite.scala | 1 - .../kyuubi/yarn/KyuubiYarnClientSuite.scala | 3 -- pom.xml | 1 - 5 files changed, 30 insertions(+), 40 deletions(-) diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml index 94eda6713..519c73237 100644 --- a/kyuubi-server/pom.xml +++ b/kyuubi-server/pom.xml @@ -126,12 +126,6 @@ true - - - ${project.build.directory}/extra-resources - true - - org.apache.maven.plugins @@ -155,6 +149,23 @@ run + + build-info-test + generate-test-resources + + + + + + + + + + + + run + + fake-jar-test generate-test-resources @@ -243,18 +254,6 @@ build-helper-maven-plugin 3.0.0 - - add-default-sources - generate-sources - - add-source - - - - v${hive.version.short}/src/main/scala - - - add-source generate-sources @@ -288,14 +287,9 @@ ${project.build.directory}/surefire-reports . TestSuite.txt - -Xmx3g -Xss4m -XX:ReservedCodeCacheSize=512m - ${project.build.testOutputDirectory}/kyuubi-server-${project.version}.jar - - ${project.build.testOutputDirectory}/tmp - diff --git a/kyuubi-server/src/main/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClient.scala b/kyuubi-server/src/main/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClient.scala index 7506c9cdb..d8a0533bf 100644 --- a/kyuubi-server/src/main/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClient.scala +++ b/kyuubi-server/src/main/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClient.scala @@ -22,6 +22,7 @@ import java.net.URI import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import java.security.PrivilegedExceptionAction +import java.text.SimpleDateFormat import java.util.{Locale, Properties, UUID} import java.util.concurrent.Callable import java.util.zip.{ZipEntry, ZipOutputStream} @@ -129,11 +130,11 @@ private[yarn] class KyuubiYarnClient(conf: SparkConf) extends Logging { case ioe: IOException => warn("Failed to cleanup staging dir " + appStagingDir, ioe) } finally { - killApplication + killApplication() } } - def killApplication: Unit = { + def killApplication(): Unit = { try { yarnClient.killApplication(appId) } catch { @@ -290,8 +291,8 @@ private[yarn] class KyuubiYarnClient(conf: SparkConf) extends Logging { } val localResources = new HashMap[String, LocalResource] FileSystem.mkdirs(fs, appStagingDir, new FsPermission(STAGING_DIR_PERMISSION)) - val symlinkCache: Map[URI, Path] = HashMap[URI, Path]() - val statCache: Map[URI, FileStatus] = HashMap[URI, FileStatus]() + val symlinkCache = HashMap[URI, Path]() + val statCache = HashMap[URI, FileStatus]() /** * Copy the non-local file to HDFS (if not already there) and add it to the Application Master's @@ -470,15 +471,17 @@ private[yarn] class KyuubiYarnClient(conf: SparkConf) extends Logging { } private[this] val stateChecker: Callable[Boolean] = new Callable[Boolean] { - private val timeout = conf.getTimeAsMs(KyuubiConf.YARN_CONTAINER_TIMEOUT.key) + private val timeout = conf.getTimeAsMs(KyuubiConf.YARN_CONTAINER_TIMEOUT.key, "60s") import YarnApplicationState._ override def call(): Boolean = { try { val report = yarnClient.getApplicationReport(appId) - info(formatReportDetails(report)) - report.getYarnApplicationState match { + val state = report.getYarnApplicationState + if (state != ACCEPTED) info(formatReportDetails(report)) + info(s"Application report for $appId[$state]") + state match { case RUNNING => false case ACCEPTED | NEW | NEW_SAVING | SUBMITTED => if (System.currentTimeMillis - report.getStartTime < timeout) { @@ -502,18 +505,17 @@ private[yarn] class KyuubiYarnClient(conf: SparkConf) extends Logging { } def formatReportDetails(report: ApplicationReport): String = { - val details = Seq[(String, String)]( + Seq( ("Client token", Option(report.getClientToAMToken).map(_.toString).getOrElse("")), ("Diagnostics", report.getDiagnostics), ("Kyuubi server host", report.getHost), ("Bind port", report.getRpcPort.toString), ("Queue", report.getQueue), - ("Start time", report.getStartTime.toString), + ("Start time", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(report.getStartTime)), ("Final status", Option(report.getFinalApplicationStatus).map(_.toString).getOrElse("")), ("Tracking URL", report.getTrackingUrl), ("User", report.getUser) - ) - details.map { case (k, v) => + ).map { case (k, v) => val newValue = Option(v).filter(_.nonEmpty).getOrElse("N/A") s"\n\t $k: $newValue" }.mkString("") @@ -619,7 +621,6 @@ object KyuubiYarnClient { */ def startKyuubiAppMaster(): Unit = { val conf = new SparkConf() - KyuubiConf.getAllDefaults.foreach(kv => conf.setIfMissing(kv._1, kv._2)) new KyuubiYarnClient(conf).submit() } } diff --git a/kyuubi-server/src/test/scala/yaooqinn/kyuubi/packageSuite.scala b/kyuubi-server/src/test/scala/yaooqinn/kyuubi/packageSuite.scala index 3a157a654..bb393ea0d 100644 --- a/kyuubi-server/src/test/scala/yaooqinn/kyuubi/packageSuite.scala +++ b/kyuubi-server/src/test/scala/yaooqinn/kyuubi/packageSuite.scala @@ -36,7 +36,6 @@ class packageSuite extends SparkFunSuite { assert(props.getProperty("revision") === REVISION) assert(props.getProperty("user") === BUILD_USER) assert(props.getProperty("url") === REPO_URL) - assert(props.getProperty("date") === BUILD_DATE) } } diff --git a/kyuubi-server/src/test/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClientSuite.scala b/kyuubi-server/src/test/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClientSuite.scala index 6dcc1052e..4001416f6 100644 --- a/kyuubi-server/src/test/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClientSuite.scala +++ b/kyuubi-server/src/test/scala/yaooqinn/kyuubi/yarn/KyuubiYarnClientSuite.scala @@ -131,8 +131,6 @@ class KyuubiYarnClientSuite extends SparkFunSuite with Matchers with MockitoSuga test("kyuubi yarn client init") { val conf = new SparkConf() - KyuubiConf.getAllDefaults.foreach(kv => conf.setIfMissing(kv._1, kv._2)) - val client = new KyuubiYarnClient(conf) assert(ReflectUtils.getFieldValue(client, "yaooqinn$kyuubi$yarn$KyuubiYarnClient$$hadoopConf").isInstanceOf[YarnConfiguration]) @@ -242,7 +240,6 @@ class KyuubiYarnClientSuite extends SparkFunSuite with Matchers with MockitoSuga def withClientBase(f: (YarnClient, KyuubiYarnClient) => Any): Unit = { val conf = new SparkConf() - KyuubiConf.getAllDefaults.foreach(kv => conf.setIfMissing(kv._1, kv._2)) val client = new KyuubiYarnClient(conf) val yarnClient = mock[YarnClient] ReflectUtils.setFieldValue(client, diff --git a/pom.xml b/pom.xml index eee52bc8f..f928f5d0c 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,6 @@ provided org.spark-project.hive 1.2.1.spark2 - 1.2.1 provided 1.1 2.0.0-M15