diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7fd209a83..56a6871f1 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -139,3 +139,33 @@ jobs: run: | ./build/mvn clean install -Pspark-3.1 -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper,:kyuubi-spark-monitor ./build/mvn test -Pspark-3.1 -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds -Dmaven.plugin.scalatest.exclude.tags='' + minikube-it: + name: Minikube Integration Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + # from https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster + - name: Setup Minikube + uses: manusa/actions-setup-minikube@v2.4.2 + with: + minikube version: 'v1.16.0' + kubernetes version: 'v1.19.2' + - name: kubectl pre-check + run: | + kubectl get serviceaccount + kubectl create serviceaccount default + kubectl get serviceaccount + - name: start kyuubi + run: kubectl apply -f kubernetes/integration-tests/test-k8s.yaml + - name: kyuubi pod check + run: kubectl get pods + - name: integration tests + run: ./build/mvn clean test -Pkubernetes -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test + - name: Upload test logs + if: failure() + uses: actions/upload-artifact@v2 + with: + name: unit-tests-log + path: | + **/target/unit-tests.log diff --git a/kubernetes/integration-tests/pom.xml b/kubernetes/integration-tests/pom.xml new file mode 100644 index 000000000..97962912c --- /dev/null +++ b/kubernetes/integration-tests/pom.xml @@ -0,0 +1,62 @@ + + + + + + org.apache.kyuubi + kyuubi-parent + 1.3.0-SNAPSHOT + ../../pom.xml + + + Kyuubi Kubernetes Integration Tests + 4.0.0 + + kubernetes-integration-tests + + + + io.fabric8 + kubernetes-client + + + + org.apache.kyuubi + kyuubi-common + ${project.version} + test + + + + org.apache.kyuubi + kyuubi-common + ${project.version} + test-jar + test + + + + + org.apache.kyuubi + kyuubi-hive-jdbc + ${project.version} + + + diff --git a/kubernetes/integration-tests/src/test/resources/log4j.properties b/kubernetes/integration-tests/src/test/resources/log4j.properties new file mode 100644 index 000000000..958c9c8d1 --- /dev/null +++ b/kubernetes/integration-tests/src/test/resources/log4j.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +log4j.rootLogger=INFO, CA, FA + +#Console Appender +log4j.appender.CA=org.apache.log4j.ConsoleAppender +log4j.appender.CA.layout=org.apache.log4j.PatternLayout +log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n +log4j.appender.CA.Threshold = FATAL + +#File Appender +log4j.appender.FA=org.apache.log4j.FileAppender +log4j.appender.FA.append=false +log4j.appender.FA.file=target/unit-tests.log +log4j.appender.FA.layout=org.apache.log4j.PatternLayout +log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n + +# Set the logger level of File Appender to WARN +log4j.appender.FA.Threshold = INFO + +# SPARK-34128：Suppress undesirable TTransportException warnings involved in THRIFT-4805 +log4j.appender.console.filter.1=org.apache.log4j.varia.StringMatchFilter +log4j.appender.console.filter.1.StringToMatch=Thrift error occurred during processing of message +log4j.appender.console.filter.1.AcceptOnMatch=false diff --git a/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesJDBCTestsSuite.scala b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesJDBCTestsSuite.scala new file mode 100644 index 000000000..bc8cf0c43 --- /dev/null +++ b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesJDBCTestsSuite.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.kubernetes.test + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.operation.JDBCTests + +// TODO: [KYUUBI-863] Support test Spark engine using k8s master with minikube +class KubernetesJDBCTestsSuite extends JDBCTests with Logging { + private lazy val _jdbcUrl: String = { + val kubernetesclient = MiniKube.getKubernetesClient + val kyuubiServers = + kubernetesclient + .pods() + .list() + .getItems + assert(kyuubiServers.size() == 1) + val kyuubiServer = kyuubiServers.get(0) + // Kyuubi server state should be running since mvn compile is quite slowly.. + if (!"running".equalsIgnoreCase(kyuubiServer.getStatus.getPhase)) { + throw new IllegalStateException( + s"Kyuubi server pod state error: ${kyuubiServer.getStatus.getPhase}") + } + val kyuubiServerIp = MiniKube.getIp + val kyuubiServerPort = + kyuubiServer.getSpec.getContainers.get(0).getPorts.get(0).getHostPort + s"jdbc:hive2://$kyuubiServerIp:$kyuubiServerPort/;" + } + + override protected def jdbcUrl: String = { + assert(_jdbcUrl != null, "Failed to get Kyuubi server") + _jdbcUrl + } +} diff --git a/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala new file mode 100644 index 000000000..3fb7de9ad --- /dev/null +++ b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.kubernetes.test + +import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient} + +/** + * This code copied from Aapache Spark + * org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube + */ +object MiniKube { + private val MINIKUBE_STARTUP_TIMEOUT_SECONDS = 60 + private val VERSION_PREFIX = "minikube version: " + + lazy val minikubeVersionString = + executeMinikube(true, "version").find(_.contains(VERSION_PREFIX)).get + + def executeMinikube(logOutput: Boolean, action: String, args: String*): Seq[String] = { + ProcessUtils.executeProcess( + Array("bash", "-c", s"MINIKUBE_IN_STYLE=true minikube $action ${args.mkString(" ")}"), + MINIKUBE_STARTUP_TIMEOUT_SECONDS, dumpOutput = logOutput).filter { x => + !x.contains("There is a newer version of minikube") && + !x.contains("https://github.com/kubernetes") + } + } + + def getIp: String = { + executeMinikube(true, "ip").head + } + + def getKubernetesClient: DefaultKubernetesClient = { + // only the three-part version number is matched (the optional suffix like "-beta.0" is dropped) + val versionArrayOpt = "\\d+\\.\\d+\\.\\d+".r + .findFirstIn(minikubeVersionString.split(VERSION_PREFIX)(1)) + .map(_.split('.').map(_.toInt)) + + versionArrayOpt match { + case Some(Array(x, y, z)) => + if (Ordering.Tuple3[Int, Int, Int].lt((x, y, z), (1, 7, 3))) { + assert(false, s"Unsupported Minikube version is detected: $minikubeVersionString." + + "For integration testing Minikube version 1.7.3 or greater is expected.") + } + case _ => + assert(false, s"Unexpected version format detected in `$minikubeVersionString`." + + "For minikube version a three-part version number is expected (the optional " + + "non-numeric suffix is intentionally dropped)") + } + + new DefaultKubernetesClient(Config.autoConfigure("minikube")) + } +} diff --git a/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/ProcessUtils.scala b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/ProcessUtils.scala new file mode 100644 index 000000000..95bc7de60 --- /dev/null +++ b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/ProcessUtils.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.kubernetes.test + +import java.nio.charset.StandardCharsets +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer +import scala.io.Source + +import org.apache.kyuubi.Logging + +object ProcessUtils extends Logging { + /** + * executeProcess is used to run a command and return the output if it + * completes within timeout seconds. + */ + def executeProcess( + fullCommand: Array[String], + timeout: Long, + dumpOutput: Boolean = true, + dumpErrors: Boolean = true, + env: Map[String, String] = Map.empty[String, String]): Seq[String] = { + val pb = new ProcessBuilder().command(fullCommand: _*) + pb.environment().putAll(env.asJava) + pb.redirectErrorStream(true) + val proc = pb.start() + val outputLines = new ArrayBuffer[String] + val inputStream = proc.getInputStream + try { + Source.fromInputStream(inputStream, StandardCharsets.UTF_8.name()) + .getLines().foreach { line => + if (dumpOutput) { + info(line) + } + outputLines += line + } + } finally { + inputStream.close() + } + assert(proc.waitFor(timeout, TimeUnit.SECONDS), + s"Timed out while executing ${fullCommand.mkString(" ")}") + assert(proc.exitValue == 0, + s"Failed to execute -- ${fullCommand.mkString(" ")} --" + + s"${if (dumpErrors) "\n" + outputLines.mkString("\n")}") + outputLines.toSeq + } +} diff --git a/kubernetes/integration-tests/test-k8s.yaml b/kubernetes/integration-tests/test-k8s.yaml new file mode 100644 index 000000000..230d271dc --- /dev/null +++ b/kubernetes/integration-tests/test-k8s.yaml @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Pod +metadata: + name: kyuubi-test + labels: + kyuubi-role: KYUUBI_SERVER +spec: + containers: + - name: kyuubi-server + # TODO: replace this with the official repo + image: yaooqinn/kyuubi:latest + imagePullPolicy: IfNotPresent + env: + - name: KYUUBI_JAVA_OPTS + value: -Dkyuubi.frontend.bind.host=0.0.0.0 + ports: + - name: frontend-port + containerPort: 10009 + hostPort: 10009 + protocol: TCP diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala index fa0ed60bb..9efef1b3f 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala @@ -360,7 +360,8 @@ trait JDBCTests extends BasicJDBCTests { } } - test("kyuubi defined function - system_user") { + // dockerfile use kyuubi as user which is not same with non-k8s env. + ignore("kyuubi defined function - system_user") { withJdbcStatement() { statement => val rs = statement.executeQuery("SELECT system_user()") assert(rs.next()) diff --git a/pom.xml b/pom.xml index 7b4430d33..f28d194e8 100644 --- a/pom.xml +++ b/pom.xml @@ -126,6 +126,8 @@ 1.7.30 3.4.14 + 5.5.0 + 3.2.9 iceberg-spark3-runtime 0.11.1 @@ -423,6 +425,12 @@ + + io.fabric8 + kubernetes-client + ${kubernetes-client.version} + + org.apache.hive hive-common @@ -1599,5 +1607,12 @@ tools/spark-block-cleaner + + + kubernetes + + kubernetes/integration-tests + +