diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index ca0fcecbc..3ed0adc29 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -355,17 +355,28 @@ jobs: run: | kubectl get serviceaccount kubectl create serviceaccount kyuubi + kubectl create clusterrolebinding kyuubi-role --clusterrole=edit --serviceaccount=default:kyuubi kubectl get serviceaccount - name: start kyuubi run: kubectl apply -f integration-tests/kyuubi-kubernetes-it/src/test/resources/kyuubi-server.yaml - name: kyuubi pod check - run: kubectl get pods + run: | + kubectl get pods + kubectl describe pods kyuubi-test - name: integration tests run: >- ./build/mvn ${MVN_OPT} clean install -pl integration-tests/kyuubi-kubernetes-it -am -Pkubernetes-it -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.deployment,org.apache.kyuubi.kubernetes.test.KubernetesUtilsTest + - name: Cat kyuubi server log + if: failure() + run: kubectl logs kyuubi-test + - name: Cat spark driver log + if: failure() + run: | + kubectl get pods + kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl logs {} - name: Upload test logs if: failure() uses: actions/upload-artifact@v3 diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala index 161011624..3a736ce48 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.kubernetes.test +import io.fabric8.kubernetes.api.model.Pod import io.fabric8.kubernetes.client.DefaultKubernetesClient import org.apache.kyuubi.KyuubiFunSuite @@ -25,26 +26,26 @@ trait WithKyuubiServerOnKubernetes extends KyuubiFunSuite { protected def connectionConf: Map[String, String] = Map.empty private val miniKubernetesClient: DefaultKubernetesClient = MiniKube.getKubernetesClient - protected def getJdbcUrl: String = { - val kyuubiServers = - miniKubernetesClient.pods().list().getItems - assert(kyuubiServers.size() == 1) - val kyuubiServer = kyuubiServers.get(0) + lazy val kyuubiPod: Pod = miniKubernetesClient.pods().withName("kyuubi-test").get() + lazy val kyuubiServerIp: String = kyuubiPod.getStatus.getPodIP + lazy val miniKubeIp: String = MiniKube.getIp + lazy val miniKubeApiMaster: String = miniKubernetesClient.getMasterUrl.toString + + protected def getJdbcUrl(connectionConf: Map[String, String]): String = { // Kyuubi server state should be running since mvn compile is quite slowly.. - if (!"running".equalsIgnoreCase(kyuubiServer.getStatus.getPhase)) { + if (!"running".equalsIgnoreCase(kyuubiPod.getStatus.getPhase)) { val log = miniKubernetesClient .pods() - .withName(kyuubiServer.getMetadata.getName) + .withName(kyuubiPod.getMetadata.getName) .getLog throw new IllegalStateException( - s"Kyuubi server pod state error: ${kyuubiServer.getStatus.getPhase}, log:\n$log") + s"Kyuubi server pod state error: ${kyuubiPod.getStatus.getPhase}, log:\n$log") } - val kyuubiServerIp = MiniKube.getIp + val kyuubiServerIp = miniKubeIp val kyuubiServerPort = - kyuubiServer.getSpec.getContainers.get(0).getPorts.get(0).getHostPort - s"jdbc:hive2://$kyuubiServerIp:$kyuubiServerPort/;" + kyuubiPod.getSpec.getContainers.get(0).getPorts.get(0).getHostPort + val connectStr = connectionConf.map(kv => kv._1 + "=" + kv._2).mkString("#", ";", "") + s"jdbc:hive2://$kyuubiServerIp:$kyuubiServerPort/;$connectStr" } - - def getMiniKubeApiMaster: String = miniKubernetesClient.getMasterUrl.toString } diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index c45c8ff22..948a696b9 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -18,6 +18,8 @@ package org.apache.kyuubi.kubernetes.test.deployment import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.hadoop.fs.permission.{FsAction, FsPermission} import org.apache.hadoop.net.NetUtils import org.apache.kyuubi.{Utils, WithSimpleDFSService} @@ -42,23 +44,23 @@ class KyuubiOnKubernetesWithLocalSparkTestsSuite extends WithKyuubiServerOnKuber super.connectionConf ++ Map("spark.master" -> "local", "spark.executor.instances" -> "1") } - override protected def jdbcUrl: String = getJdbcUrl + override protected def jdbcUrl: String = getJdbcUrl(connectionConf) + + override protected lazy val user: String = "local" } -class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes - with SparkQueryTests { +class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes { override protected def connectionConf: Map[String, String] = { super.connectionConf ++ Map( - "spark.master" -> s"k8s://$getMiniKubeApiMaster", + "spark.master" -> s"k8s://$miniKubeApiMaster", + "spark.kubernetes.container.image" -> "apache/spark:3.3.1", "spark.executor.memory" -> "512M", - "spark.driver.memory" -> "512M", + "spark.driver.memory" -> "1024M", "spark.kubernetes.driver.request.cores" -> "250m", "spark.kubernetes.executor.request.cores" -> "250m", "spark.executor.instances" -> "1") } - - override protected def jdbcUrl: String = getJdbcUrl } /** @@ -72,10 +74,17 @@ class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes * ------------ ------------------------------------------------- --------------------- */ class KyuubiOnKubernetesWithClientSparkTestsSuite - extends KyuubiOnKubernetesWithSparkTestsBase { + extends KyuubiOnKubernetesWithSparkTestsBase with SparkQueryTests { override protected def connectionConf: Map[String, String] = { - super.connectionConf ++ Map("spark.submit.deployMode" -> "client") + super.connectionConf ++ Map( + "spark.submit.deployMode" -> "client", + "spark.driver.host" -> kyuubiServerIp, + "kyuubi.frontend.connection.url.use.hostname" -> "false") } + + override protected def jdbcUrl: String = getJdbcUrl(connectionConf) + + override protected lazy val user: String = "client" } /** @@ -89,11 +98,22 @@ class KyuubiOnKubernetesWithClientSparkTestsSuite * ---------- ----------------- ----------------------------- --------------------- */ class KyuubiOnKubernetesWithClusterSparkTestsSuite - extends KyuubiOnKubernetesWithSparkTestsBase with WithSimpleDFSService { + extends KyuubiOnKubernetesWithSparkTestsBase with WithSimpleDFSService with SparkQueryTests { private val localhostAddress = Utils.findLocalInetAddress.getHostAddress private val driverTemplate = Thread.currentThread().getContextClassLoader.getResource("driver.yml") + override def beforeAll(): Unit = { + super.beforeAll() + val fs = FileSystem.get(getHadoopConf) + fs.mkdirs( + new Path("/spark"), + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) + fs.setPermission(new Path("/"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) + fs.setPermission(new Path("/spark"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) + fs.copyFromLocalFile(new Path(driverTemplate.getPath), new Path("/spark/driver.yml")) + } + override val hadoopConf: Configuration = { val hdfsConf: Configuration = new Configuration() hdfsConf.set("dfs.namenode.rpc-bind-host", "0.0.0.0") @@ -103,6 +123,8 @@ class KyuubiOnKubernetesWithClusterSparkTestsSuite // spark use 185 as userid in docker hdfsConf.set("hadoop.proxyuser.185.groups", "*") hdfsConf.set("hadoop.proxyuser.185.hosts", "*") + hdfsConf.set("hadoop.proxyuser.kyuubi.groups", "*") + hdfsConf.set("hadoop.proxyuser.kyuubi.hosts", "*") hdfsConf } @@ -112,9 +134,14 @@ class KyuubiOnKubernetesWithClusterSparkTestsSuite "spark.submit.deployMode" -> "cluster", "spark.kubernetes.file.upload.path" -> s"hdfs://$localhostAddress:$getDFSPort/spark", "spark.hadoop.dfs.client.use.datanode.hostname" -> "true", - "spark.kubernetes.authenticate.driver.serviceAccountName" -> "spark", - "spark.kubernetes.driver.podTemplateFile" -> driverTemplate.getPath, - ZK_CLIENT_PORT_ADDRESS.key -> localhostAddress, - FRONTEND_THRIFT_BINARY_BIND_HOST.key -> localhostAddress) + "spark.kubernetes.authenticate.driver.serviceAccountName" -> "kyuubi", + "spark.kubernetes.driver.podTemplateFile" -> + s"hdfs://$localhostAddress:$getDFSPort/spark/driver.yml", + "spark.kyuubi.frontend.thrift.binary.bind.host" -> miniKubeIp, + "spark.kyuubi.ha.addresses" -> s"$kyuubiServerIp:2181", + ZK_CLIENT_PORT_ADDRESS.key -> kyuubiServerIp, + FRONTEND_THRIFT_BINARY_BIND_HOST.key -> kyuubiServerIp) } + + override protected def jdbcUrl: String = getJdbcUrl(connectionConf) }