apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #2381] [Test] Add Kyuubi on k8s With Spark on k8s client deploy-mode unit test

Browse Source 
### _Why are the changes needed?_

Add Kyuubi on k8s With Spark on k8s client deploy-mode unit test

### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #2381 from zwangsheng/improve/add-kyuubi-k8s-spark-client.

Closes #2381

4f4d72f5 [zwangsheng] fix npe
623b3568 [zwangsheng] fix style
03ab628f [zwangsheng] add kyuubi on k8s spark client test

Authored-by: zwangsheng <2213335496@qq.com>
Signed-off-by: ulysses-you <ulyssesyou@apache.org>
This commit is contained in:
zwangsheng 2022-04-18 10:59:29 +08:00 committed by ulysses-you
parent eb7ad512e1
commit 62db92f72b
No known key found for this signature in database
GPG Key ID: 4C500BC62D576766
2 changed files with 29 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala
View File

@ -25,20 +25,13 @@ import org.apache.kyuubi.config.KyuubiConf
trait WithKyuubiServerOnKubernetes extends WithKyuubiServer {
protected val kyuubiServerConf: KyuubiConf = KyuubiConf()
protected val connectionConf: Map[String, String]
private var miniKubernetesClient: DefaultKubernetesClient = _
private val miniKubernetesClient: DefaultKubernetesClient = MiniKube.getKubernetesClient
final override protected lazy val conf: KyuubiConf = {
connectionConf.foreach { case (k, v) => kyuubiServerConf.set(k, v) }
kyuubiServerConf
}
override def beforeAll(): Unit = {
miniKubernetesClient = MiniKube.getKubernetesClient
super.beforeAll()
}
override def afterAll(): Unit = super.afterAll()
override protected def getJdbcUrl: String = {
val kyuubiServers =
miniKubernetesClient.pods().list().getItems
@ -59,4 +52,6 @@ trait WithKyuubiServerOnKubernetes extends WithKyuubiServer {
kyuubiServer.getSpec.getContainers.get(0).getPorts.get(0).getHostPort
s"jdbc:hive2://$kyuubiServerIp:$kyuubiServerPort/;"
}
def getMiniKubeApiMaster: String = miniKubernetesClient.getMasterUrl.toString
}

27
integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
View File

@ -21,7 +21,7 @@ import org.apache.kyuubi.kubernetes.test.WithKyuubiServerOnKubernetes
import org.apache.kyuubi.operation.SparkQueryTests
/**
* This test is for Kyuubi Server on Kubernetes with Spark engine:
* This test is for Kyuubi Server on Kubernetes with Spark engine local deploy-mode:
*
* Real World Kubernetes Pod
* ------------ -----------------------------------------------------
@ -38,3 +38,28 @@ class KyuubiOnKubernetesWithLocalSparkTestsSuite extends WithKyuubiServerOnKuber
override protected def jdbcUrl: String = getJdbcUrl
}
/**
* This test is for Kyuubi Server on Kubernetes with Spark engine On Kubernetes client deploy-mode:
*
* Real World Kubernetes Pod
* ------------ ------------------------------------------------- ---------------------
* | | JDBC | | | |
* | Client | ----> | Kyuubi Server --> Spark Engine (client mode) | --> | Spark Executors |
* | | | | | |
* ------------ ------------------------------------------------- ---------------------
*/
class KyuubiOnKubernetesWithClientSparkOnKubernetesTestsSuite extends WithKyuubiServerOnKubernetes
with SparkQueryTests {
override protected val connectionConf: Map[String, String] = Map(
"spark.master" -> s"k8s://$getMiniKubeApiMaster",
"spark.submit.deployMode" -> "client",
"spark.kubernetes.container.image" -> "apache/spark:v3.2.1",
"spark.executor.memory" -> "512M",
"spark.driver.memory" -> "512M",
"spark.kubernetes.driver.request.cores" -> "250m",
"spark.kubernetes.executor.request.cores" -> "250m",
"spark.executor.instances" -> "1")
override protected def jdbcUrl: String = getJdbcUrl
}