From b0e7c07ce9d975ecb82375751a9a0164e8e3fc06 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Thu, 29 Dec 2022 15:35:21 +0800 Subject: [PATCH] [KYUUBI #4038] [K8S][LOG]Improve print diagnostics info in kubernetes cluster deploy-mode case ### _Why are the changes needed?_ For Spark On Kubernetes Cluster Deploy-Mode, spark will set `spark.submit.deployMode=client` and `spark.kubernetes.submitInDriver=true`. Kyuubi print diagnostics about deploy-mode for spark sql engine should not only dependence on `spark.submit.deployMode`. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4038 from zwangsheng/improve/diagnostics_deploy_mode_for_k8s_cluster. Closes #4038 5149a0be [zwangsheng] Improve Diagnostics Info in Kubernetes Cluster Mode Case Authored-by: zwangsheng <2213335496@qq.com> Signed-off-by: Cheng Pan --- .../org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala index 065d5ff57..48760bda9 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala @@ -57,6 +57,13 @@ object KyuubiSparkUtil extends Logging { def engineUrl: String = globalSparkContext.getConf.getOption( "spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES") .orElse(globalSparkContext.uiWebUrl).getOrElse("") + def deployMode(sc: SparkContext): String = { + if (sc.getConf.getBoolean("spark.kubernetes.submitInDriver", false)) { + "cluster" + } else { + sc.deployMode + } + } lazy val diagnostics: String = { val sc = globalSparkContext @@ -67,7 +74,7 @@ object KyuubiSparkUtil extends Logging { | application ID: $engineId | application web UI: $engineUrl | master: ${sc.master} - | deploy mode: ${sc.deployMode} + | deploy mode: ${deployMode(sc)} | version: ${sc.version} | Start time: ${LocalDateTime.ofInstant(Instant.ofEpochMilli(sc.startTime), ZoneId.systemDefault)} | User: ${sc.sparkUser}""".stripMargin