From 65448dda04dc3a1034d3bf92997c9c190f10acf1 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 5 Nov 2018 17:40:46 +0800 Subject: [PATCH] fix #114 kyuubi should work well with spark local mode --- .../apache/spark/deploy/KyuubiSubmit.scala | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/kyuubi-server/src/main/scala/org/apache/spark/deploy/KyuubiSubmit.scala b/kyuubi-server/src/main/scala/org/apache/spark/deploy/KyuubiSubmit.scala index 13f239630..43ff99b7d 100644 --- a/kyuubi-server/src/main/scala/org/apache/spark/deploy/KyuubiSubmit.scala +++ b/kyuubi-server/src/main/scala/org/apache/spark/deploy/KyuubiSubmit.scala @@ -89,16 +89,25 @@ object KyuubiSubmit extends Logging { */ private[deploy] def prepareSubmitEnvironment(args: SparkSubmitArguments): (Seq[String], Map[String, String]) = { - // Return values + // Make sure YARN is included in our build if we're trying to use it + def checkYarnSupport: Unit = { + if (!Utils.classIsLoadable("org.apache.spark.deploy.yarn.Client")) { + printErrorAndExit( + "Could not load YARN classes. Spark may not have been compiled with YARN support.") + } + } + val childClasspath = new ArrayBuffer[String]() val sysProps = new HashMap[String, String]() args.master match { - case "yarn" => - case "yarn-client" => + case "yarn" => checkYarnSupport + case "yarn-client" | "yarn-cluster" => + checkYarnSupport printWarning(s"Master ${args.master} is deprecated since 2.0." + " Please use master \"yarn\" with specified deploy mode instead.") args.master = "yarn" + case m if m.startsWith("local") => args.master = "local" case _ => printErrorAndExit("Kyuubi only supports yarn as master.") } @@ -110,12 +119,6 @@ object KyuubiSubmit extends Logging { } - // Make sure YARN is included in our build if we're trying to use it - if (!Utils.classIsLoadable("org.apache.spark.deploy.yarn.Client")) { - printErrorAndExit( - "Could not load YARN classes. Spark may not have been compiled with YARN support.") - } - Seq( "spark.master" -> args.master, "spark.submit.deployMode" -> args.deployMode,