apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #1594] SparkProcessBuild submit add --conf spark.driver.host=

Browse Source 
<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->
detail #1594
When Kyuubi On Kubernetes submit spark engine by client mode, executor pod can not connect to driver by kyuubi pod name, which is default set.

### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #1596 from zwangsheng/kyuubi/1594.

Closes #1594

d4b41005 [zwangsheng] add comment
f7bcffb6 [zwangsheng] fix test name
581eb508 [zwangsheng] fix test name
b19a2c21 [zwangsheng] simple & add testsuit
83979935 [zwangsheng] fix
1a9ae9b1 [zwangsheng] fix doc
d857eac7 [zwangsheng] Merge branch 'kyuubi/1594' of https://github.com/zwangsheng/incubator-kyuubi into kyuubi/1594
b723f4ed [zwangsheng] retest
8210de56 [zwangsheng] make import normal
92ec9cf4 [zwangsheng] 1594

Authored-by: zwangsheng <2213335496@qq.com>
Signed-off-by: Kent Yao <yao@apache.org>
This commit is contained in:
zwangsheng 2021-12-31 19:25:18 +08:00 committed by Kent Yao
parent 3eaa1d86c8
commit 872551d3aa
No known key found for this signature in database
GPG Key ID: F7051850A0AF904D
2 changed files with 36 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
View File

@ -27,7 +27,7 @@ import org.apache.hadoop.security.UserGroupInformation
import org.apache.kyuubi._
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAIN_RESOURCE
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_SPARK_MAIN_RESOURCE, FRONTEND_THRIFT_BINARY_BIND_HOST}
import org.apache.kyuubi.engine.ProcBuilder
import org.apache.kyuubi.ha.HighAvailabilityConf
import org.apache.kyuubi.ha.client.ZooKeeperAuthTypes
@ -157,6 +157,20 @@ class SparkProcessBuilder(
buffer += s"$newKey=$v"
}
/**
* Kyuubi respect user setting config, if user set `spark.driver.host`, will pass it on.
* If user don't set this, will use thrift binary bind host to set.
* Kyuubi wants the Engine to bind hostName or IP with Kyuubi.
* Spark driver will pass this configuration as the driver-url to the executors
* to build RPC communication.
*/
if (!allConf.contains("spark.driver.host")) {
conf.get(FRONTEND_THRIFT_BINARY_BIND_HOST).foreach(host => {
buffer += CONF
buffer += s"spark.driver.host=${host}"
})
}
// iff the keytab is specified, PROXY_USER is not supported
if (!useKeytab()) {
buffer += PROXY_USER

23
kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
View File

@ -26,8 +26,7 @@ import org.scalatest.time.SpanSugar._
import org.apache.kyuubi.{KerberizedTestHelper, KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.ENGINE_LOG_TIMEOUT
import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAIN_RESOURCE
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_LOG_TIMEOUT, ENGINE_SPARK_MAIN_RESOURCE, FRONTEND_THRIFT_BINARY_BIND_HOST}
import org.apache.kyuubi.ha.HighAvailabilityConf
import org.apache.kyuubi.ha.client.ZooKeeperAuthTypes
import org.apache.kyuubi.service.ServiceUtils
@ -272,6 +271,26 @@ class SparkProcessBuilderSuite extends KerberizedTestHelper {
assert(b1.toString.contains(s"--conf spark.files=$testKeytab"))
}
test("engine bind on host name or IP with Kyuubi") {
val conf = KyuubiConf()
conf.set(FRONTEND_THRIFT_BINARY_BIND_HOST.key, "kyuubi-example")
val builder = new SparkProcessBuilder("test", conf)
assert(builder.toString.contains("--conf spark.driver.host=kyuubi-example"))
}
test("respect to user set config") {
val conf = KyuubiConf()
conf.set(FRONTEND_THRIFT_BINARY_BIND_HOST.key, "kyuubi-example")
conf.set("spark.driver.host", "spark-example")
val builder = new SparkProcessBuilder("test", conf)
assertResult(false) {
builder.toString.contains("--conf spark.driver.host=kyuubi-example")
}
assert(builder.toString.contains("--conf spark.driver.host=spark-example"))
}
}
class FakeSparkProcessBuilder(config: KyuubiConf)