From 872551d3aaf851edfd59b2bcfa4a5ce2be4710ef Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Fri, 31 Dec 2021 19:25:18 +0800
Subject: [PATCH] [KYUUBI #1594] SparkProcessBuild submit add --conf
 spark.driver.host=

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->
detail #1594
When Kyuubi On Kubernetes submit spark engine by client mode, executor pod can not connect to driver by kyuubi pod name, which is default set.

### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #1596 from zwangsheng/kyuubi/1594.

Closes #1594

d4b41005 [zwangsheng] add comment
f7bcffb6 [zwangsheng] fix test name
581eb508 [zwangsheng] fix test name
b19a2c21 [zwangsheng] simple & add testsuit
83979935 [zwangsheng] fix
1a9ae9b1 [zwangsheng] fix doc
d857eac7 [zwangsheng] Merge branch 'kyuubi/1594' of https://github.com/zwangsheng/incubator-kyuubi into kyuubi/1594
b723f4ed [zwangsheng] retest
8210de56 [zwangsheng] make import normal
92ec9cf4 [zwangsheng] 1594

Authored-by: zwangsheng <2213335496@qq.com>
Signed-off-by: Kent Yao <yao@apache.org>
---
 .../engine/spark/SparkProcessBuilder.scala    | 16 ++++++++++++-
 .../spark/SparkProcessBuilderSuite.scala      | 23 +++++++++++++++++--
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
index b78a5ad7a..e3c491b64 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.security.UserGroupInformation
 
 import org.apache.kyuubi._
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAIN_RESOURCE
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_SPARK_MAIN_RESOURCE, FRONTEND_THRIFT_BINARY_BIND_HOST}
 import org.apache.kyuubi.engine.ProcBuilder
 import org.apache.kyuubi.ha.HighAvailabilityConf
 import org.apache.kyuubi.ha.client.ZooKeeperAuthTypes
@@ -157,6 +157,20 @@ class SparkProcessBuilder(
       buffer += s"$newKey=$v"
     }
 
+    /**
+     * Kyuubi respect user setting config, if user set `spark.driver.host`, will pass it on.
+     * If user don't set this, will use thrift binary bind host to set.
+     * Kyuubi wants the Engine to bind hostName or IP with Kyuubi.
+     * Spark driver will pass this configuration as the driver-url to the executors
+     * to build RPC communication.
+     */
+    if (!allConf.contains("spark.driver.host")) {
+      conf.get(FRONTEND_THRIFT_BINARY_BIND_HOST).foreach(host => {
+        buffer += CONF
+        buffer += s"spark.driver.host=${host}"
+      })
+    }
+
     // iff the keytab is specified, PROXY_USER is not supported
     if (!useKeytab()) {
       buffer += PROXY_USER
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
index d50b75aad..79fb21f96 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
@@ -26,8 +26,7 @@ import org.scalatest.time.SpanSugar._
 
 import org.apache.kyuubi.{KerberizedTestHelper, KyuubiSQLException, Utils}
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.ENGINE_LOG_TIMEOUT
-import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAIN_RESOURCE
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_LOG_TIMEOUT, ENGINE_SPARK_MAIN_RESOURCE, FRONTEND_THRIFT_BINARY_BIND_HOST}
 import org.apache.kyuubi.ha.HighAvailabilityConf
 import org.apache.kyuubi.ha.client.ZooKeeperAuthTypes
 import org.apache.kyuubi.service.ServiceUtils
@@ -272,6 +271,26 @@ class SparkProcessBuilderSuite extends KerberizedTestHelper {
     assert(b1.toString.contains(s"--conf spark.files=$testKeytab"))
 
   }
+
+  test("engine bind on host name or IP with Kyuubi") {
+    val conf = KyuubiConf()
+    conf.set(FRONTEND_THRIFT_BINARY_BIND_HOST.key, "kyuubi-example")
+
+    val builder = new SparkProcessBuilder("test", conf)
+    assert(builder.toString.contains("--conf spark.driver.host=kyuubi-example"))
+  }
+
+  test("respect to user set config") {
+    val conf = KyuubiConf()
+    conf.set(FRONTEND_THRIFT_BINARY_BIND_HOST.key, "kyuubi-example")
+    conf.set("spark.driver.host", "spark-example")
+
+    val builder = new SparkProcessBuilder("test", conf)
+    assertResult(false) {
+      builder.toString.contains("--conf spark.driver.host=kyuubi-example")
+    }
+    assert(builder.toString.contains("--conf spark.driver.host=spark-example"))
+  }
 }
 
 class FakeSparkProcessBuilder(config: KyuubiConf)