From 7631e7d8b8ca66b635f9d761c41d40e2bb73ea10 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Fri, 21 Jul 2023 11:21:07 +0800 Subject: [PATCH] [KYUUBI #5009] Pass Spark Engine Log Path to Spark Conf ### _Why are the changes needed?_ Close #5009 When Kyuubi Server Log is Huge, it's difficult to find `Spark Engine Log Path` in logs. Here pass the path to spark conf, user can find engine log path in spark ui or spark history server. Submit Command Like: ```shell XXXX/bin/spark-submit \ --class org.apache.kyuubi.engine.spark.SparkSQLEngine \ --conf spark.kyuubi.engine.engineLog.path=XXXX/kyuubi-spark-sql-engine.log.0 \ --proxy-user kyuubi XXXX/target/kyuubi-spark-sql-engine_2.12-1.8.0-SNAPSHOT.jar ``` ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request Closes #5011 from zwangsheng/KYUUBI_5009. Closes #5009 36c772209 [zwangsheng] fix compile 1c20f9264 [zwangsheng] retest 70568c758 [zwangsheng] Fix Unit Test 2bc465740 [zwangsheng] try to fix unit test 2197b3503 [zwangsheng] Narrow the scope of access a44eefc5c [zwangsheng] [KYUUBI #5009]Pass Spark Engine Log Path to Spark COnf Authored-by: zwangsheng <2213335496@qq.com> Signed-off-by: Cheng Pan --- .../scala/org/apache/kyuubi/engine/ProcBuilder.scala | 2 ++ .../engine/spark/SparkBatchProcessBuilder.scala | 4 ++-- .../kyuubi/engine/spark/SparkProcessBuilder.scala | 11 ++++++++--- .../engine/spark/SparkProcessBuilderSuite.scala | 11 ++++++++++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala index d0e6bae6b..f1e49d687 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala @@ -359,4 +359,6 @@ object ProcBuilder extends Logging { private val PROC_BUILD_LOGGER = new NamedThreadFactory("process-logger-capture", daemon = true) private val UNCAUGHT_ERROR = new RuntimeException("Uncaught error") + + private[engine] val KYUUBI_ENGINE_LOG_PATH_KEY = "kyuubi.engine.engineLog.path" } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala index 0d20068de..7f1be93b5 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala @@ -36,7 +36,7 @@ class SparkBatchProcessBuilder( extends SparkProcessBuilder(proxyUser, conf, batchId, extraEngineLog) { import SparkProcessBuilder._ - override protected val commands: Array[String] = { + override protected lazy val commands: Array[String] = { val buffer = new ArrayBuffer[String]() buffer += executable Option(mainClass).foreach { cla => @@ -51,7 +51,7 @@ class SparkBatchProcessBuilder( // tag batch application KyuubiApplicationManager.tagApplication(batchId, "spark", clusterManager(), batchKyuubiConf) - (batchKyuubiConf.getAll ++ sparkAppNameConf()).foreach { case (k, v) => + (batchKyuubiConf.getAll ++ sparkAppNameConf() ++ engineLogPathConf()).foreach { case (k, v) => buffer += CONF buffer += s"${convertConfigKey(k)}=$v" } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index 5bfe506da..5998d5d4e 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -30,6 +30,7 @@ import org.apache.kyuubi._ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.engine.{ApplicationManagerInfo, KyuubiApplicationManager, ProcBuilder} import org.apache.kyuubi.engine.KubernetesApplicationOperation.{KUBERNETES_SERVICE_HOST, KUBERNETES_SERVICE_PORT} +import org.apache.kyuubi.engine.ProcBuilder.KYUUBI_ENGINE_LOG_PATH_KEY import org.apache.kyuubi.ha.HighAvailabilityConf import org.apache.kyuubi.ha.client.AuthTypes import org.apache.kyuubi.operation.log.OperationLog @@ -99,7 +100,7 @@ class SparkProcessBuilder( } } - override protected val commands: Array[String] = { + override protected lazy val commands: Array[String] = { // complete `spark.master` if absent on kubernetes completeMasterUrl(conf) @@ -116,8 +117,8 @@ class SparkProcessBuilder( == AuthTypes.KERBEROS) { allConf = allConf ++ zkAuthKeytabFileConf(allConf) } - - allConf.foreach { case (k, v) => + // pass spark engine log path to spark conf + (allConf ++ engineLogPathConf).foreach { case (k, v) => buffer += CONF buffer += s"${convertConfigKey(k)}=$v" } @@ -244,6 +245,10 @@ class SparkProcessBuilder( } } } + + private[spark] def engineLogPathConf(): Map[String, String] = { + Map(KYUUBI_ENGINE_LOG_PATH_KEY -> engineLog.getAbsolutePath) + } } object SparkProcessBuilder { diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala index ad8324c85..cd549c0f3 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala @@ -29,6 +29,8 @@ import org.scalatestplus.mockito.MockitoSugar import org.apache.kyuubi.{KerberizedTestHelper, KyuubiSQLException, Utils} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.{ENGINE_LOG_TIMEOUT, ENGINE_SPARK_MAIN_RESOURCE} +import org.apache.kyuubi.engine.ProcBuilder.KYUUBI_ENGINE_LOG_PATH_KEY +import org.apache.kyuubi.engine.spark.SparkProcessBuilder.CONF import org.apache.kyuubi.ha.HighAvailabilityConf import org.apache.kyuubi.ha.client.AuthTypes import org.apache.kyuubi.service.ServiceUtils @@ -296,9 +298,16 @@ class SparkProcessBuilderSuite extends KerberizedTestHelper with MockitoSugar { assert(!c3.contains(s"spark.kubernetes.driverEnv.SPARK_USER_NAME=$proxyName")) assert(!c3.contains(s"spark.executorEnv.SPARK_USER_NAME=$proxyName")) } + + test("[KYUUBI #5009] Test pass spark engine log path to spark conf") { + val b1 = new SparkProcessBuilder("kyuubi", conf) + assert( + b1.toString.contains( + s"$CONF spark.$KYUUBI_ENGINE_LOG_PATH_KEY=${b1.engineLog.getAbsolutePath}")) + } } class FakeSparkProcessBuilder(config: KyuubiConf) extends SparkProcessBuilder("fake", config) { - override protected val commands: Array[String] = Array("ls") + override protected lazy val commands: Array[String] = Array("ls") }