<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> * Use github action to install `MiniKube` * Start Kyuubi Server as Pod with `MiniKube` * Add a new module `kubernetes-integration-tests` to run basic JDBC test Since there exists many `MiniKube` app in github market, this PR use [Setup Minikube Kubernetes Cluster](https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster) , we can change this if necessary in future. Note that, this PR is for Kyuubi test with k8s not for Apache Spark. ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request Closes #853 from ulysses-you/k8s-it. Closes #444 Closes #854 4a73a8fa [ulysses-you] ignore a501482d [ulysses-you] ignore 242f60c4 [ulysses-you] todo 994e52e4 [ulysses-you] pom 3f60fb2e [ulysses-you] Merge branch 'master' of https://github.com/NetEase/kyuubi into k8s-it 3febdcd2 [ulysses-you] fix f564792b [ulysses-you] Add minikuube for k8s integration test Authored-by: ulysses-you <ulyssesyou18@gmail.com> Signed-off-by: Kent Yao <yao@apache.org>
This commit is contained in:
ulysses-you
Kent Yao
parent
286e3fbcc1
commit
536e5fde72
30
.github/workflows/master.yml
vendored
30
.github/workflows/master.yml
vendored
@ -139,3 +139,33 @@ jobs:
|
||||
run: |
|
||||
./build/mvn clean install -Pspark-3.1 -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper,:kyuubi-spark-monitor
|
||||
./build/mvn test -Pspark-3.1 -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds -Dmaven.plugin.scalatest.exclude.tags=''
|
||||
minikube-it:
|
||||
name: Minikube Integration Test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
# from https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster
|
||||
- name: Setup Minikube
|
||||
uses: manusa/actions-setup-minikube@v2.4.2
|
||||
with:
|
||||
minikube version: 'v1.16.0'
|
||||
kubernetes version: 'v1.19.2'
|
||||
- name: kubectl pre-check
|
||||
run: |
|
||||
kubectl get serviceaccount
|
||||
kubectl create serviceaccount default
|
||||
kubectl get serviceaccount
|
||||
- name: start kyuubi
|
||||
run: kubectl apply -f kubernetes/integration-tests/test-k8s.yaml
|
||||
- name: kyuubi pod check
|
||||
run: kubectl get pods
|
||||
- name: integration tests
|
||||
run: ./build/mvn clean test -Pkubernetes -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test
|
||||
- name: Upload test logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: unit-tests-log
|
||||
path: |
|
||||
**/target/unit-tests.log
|
||||
|
||||
62
kubernetes/integration-tests/pom.xml
Normal file
62
kubernetes/integration-tests/pom.xml
Normal file
@ -0,0 +1,62 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>kyuubi-parent</artifactId>
|
||||
<version>1.3.0-SNAPSHOT</version>
|
||||
<relativePath>../../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<name>Kyuubi Kubernetes Integration Tests</name>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>kubernetes-integration-tests</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>io.fabric8</groupId>
|
||||
<artifactId>kubernetes-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>kyuubi-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>kyuubi-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- for hive driver related dependency -->
|
||||
<dependency>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>kyuubi-hive-jdbc</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@ -0,0 +1,40 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Set everything to be logged to the file target/unit-tests.log
|
||||
log4j.rootLogger=INFO, CA, FA
|
||||
|
||||
#Console Appender
|
||||
log4j.appender.CA=org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.CA.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
|
||||
log4j.appender.CA.Threshold = FATAL
|
||||
|
||||
#File Appender
|
||||
log4j.appender.FA=org.apache.log4j.FileAppender
|
||||
log4j.appender.FA.append=false
|
||||
log4j.appender.FA.file=target/unit-tests.log
|
||||
log4j.appender.FA.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n
|
||||
|
||||
# Set the logger level of File Appender to WARN
|
||||
log4j.appender.FA.Threshold = INFO
|
||||
|
||||
# SPARK-34128：Suppress undesirable TTransportException warnings involved in THRIFT-4805
|
||||
log4j.appender.console.filter.1=org.apache.log4j.varia.StringMatchFilter
|
||||
log4j.appender.console.filter.1.StringToMatch=Thrift error occurred during processing of message
|
||||
log4j.appender.console.filter.1.AcceptOnMatch=false
|
||||
@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kyuubi.kubernetes.test
|
||||
|
||||
import org.apache.kyuubi.Logging
|
||||
import org.apache.kyuubi.operation.JDBCTests
|
||||
|
||||
// TODO: [KYUUBI-863] Support test Spark engine using k8s master with minikube
|
||||
class KubernetesJDBCTestsSuite extends JDBCTests with Logging {
|
||||
private lazy val _jdbcUrl: String = {
|
||||
val kubernetesclient = MiniKube.getKubernetesClient
|
||||
val kyuubiServers =
|
||||
kubernetesclient
|
||||
.pods()
|
||||
.list()
|
||||
.getItems
|
||||
assert(kyuubiServers.size() == 1)
|
||||
val kyuubiServer = kyuubiServers.get(0)
|
||||
// Kyuubi server state should be running since mvn compile is quite slowly..
|
||||
if (!"running".equalsIgnoreCase(kyuubiServer.getStatus.getPhase)) {
|
||||
throw new IllegalStateException(
|
||||
s"Kyuubi server pod state error: ${kyuubiServer.getStatus.getPhase}")
|
||||
}
|
||||
val kyuubiServerIp = MiniKube.getIp
|
||||
val kyuubiServerPort =
|
||||
kyuubiServer.getSpec.getContainers.get(0).getPorts.get(0).getHostPort
|
||||
s"jdbc:hive2://$kyuubiServerIp:$kyuubiServerPort/;"
|
||||
}
|
||||
|
||||
override protected def jdbcUrl: String = {
|
||||
assert(_jdbcUrl != null, "Failed to get Kyuubi server")
|
||||
_jdbcUrl
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kyuubi.kubernetes.test
|
||||
|
||||
import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient}
|
||||
|
||||
/**
|
||||
* This code copied from Aapache Spark
|
||||
* org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube
|
||||
*/
|
||||
object MiniKube {
|
||||
private val MINIKUBE_STARTUP_TIMEOUT_SECONDS = 60
|
||||
private val VERSION_PREFIX = "minikube version: "
|
||||
|
||||
lazy val minikubeVersionString =
|
||||
executeMinikube(true, "version").find(_.contains(VERSION_PREFIX)).get
|
||||
|
||||
def executeMinikube(logOutput: Boolean, action: String, args: String*): Seq[String] = {
|
||||
ProcessUtils.executeProcess(
|
||||
Array("bash", "-c", s"MINIKUBE_IN_STYLE=true minikube $action ${args.mkString(" ")}"),
|
||||
MINIKUBE_STARTUP_TIMEOUT_SECONDS, dumpOutput = logOutput).filter { x =>
|
||||
!x.contains("There is a newer version of minikube") &&
|
||||
!x.contains("https://github.com/kubernetes")
|
||||
}
|
||||
}
|
||||
|
||||
def getIp: String = {
|
||||
executeMinikube(true, "ip").head
|
||||
}
|
||||
|
||||
def getKubernetesClient: DefaultKubernetesClient = {
|
||||
// only the three-part version number is matched (the optional suffix like "-beta.0" is dropped)
|
||||
val versionArrayOpt = "\\d+\\.\\d+\\.\\d+".r
|
||||
.findFirstIn(minikubeVersionString.split(VERSION_PREFIX)(1))
|
||||
.map(_.split('.').map(_.toInt))
|
||||
|
||||
versionArrayOpt match {
|
||||
case Some(Array(x, y, z)) =>
|
||||
if (Ordering.Tuple3[Int, Int, Int].lt((x, y, z), (1, 7, 3))) {
|
||||
assert(false, s"Unsupported Minikube version is detected: $minikubeVersionString." +
|
||||
"For integration testing Minikube version 1.7.3 or greater is expected.")
|
||||
}
|
||||
case _ =>
|
||||
assert(false, s"Unexpected version format detected in `$minikubeVersionString`." +
|
||||
"For minikube version a three-part version number is expected (the optional " +
|
||||
"non-numeric suffix is intentionally dropped)")
|
||||
}
|
||||
|
||||
new DefaultKubernetesClient(Config.autoConfigure("minikube"))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,63 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.kyuubi.kubernetes.test
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable.ArrayBuffer
|
||||
import scala.io.Source
|
||||
|
||||
import org.apache.kyuubi.Logging
|
||||
|
||||
object ProcessUtils extends Logging {
|
||||
/**
|
||||
* executeProcess is used to run a command and return the output if it
|
||||
* completes within timeout seconds.
|
||||
*/
|
||||
def executeProcess(
|
||||
fullCommand: Array[String],
|
||||
timeout: Long,
|
||||
dumpOutput: Boolean = true,
|
||||
dumpErrors: Boolean = true,
|
||||
env: Map[String, String] = Map.empty[String, String]): Seq[String] = {
|
||||
val pb = new ProcessBuilder().command(fullCommand: _*)
|
||||
pb.environment().putAll(env.asJava)
|
||||
pb.redirectErrorStream(true)
|
||||
val proc = pb.start()
|
||||
val outputLines = new ArrayBuffer[String]
|
||||
val inputStream = proc.getInputStream
|
||||
try {
|
||||
Source.fromInputStream(inputStream, StandardCharsets.UTF_8.name())
|
||||
.getLines().foreach { line =>
|
||||
if (dumpOutput) {
|
||||
info(line)
|
||||
}
|
||||
outputLines += line
|
||||
}
|
||||
} finally {
|
||||
inputStream.close()
|
||||
}
|
||||
assert(proc.waitFor(timeout, TimeUnit.SECONDS),
|
||||
s"Timed out while executing ${fullCommand.mkString(" ")}")
|
||||
assert(proc.exitValue == 0,
|
||||
s"Failed to execute -- ${fullCommand.mkString(" ")} --" +
|
||||
s"${if (dumpErrors) "\n" + outputLines.mkString("\n")}")
|
||||
outputLines.toSeq
|
||||
}
|
||||
}
|
||||
37
kubernetes/integration-tests/test-k8s.yaml
Normal file
37
kubernetes/integration-tests/test-k8s.yaml
Normal file
@ -0,0 +1,37 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: kyuubi-test
|
||||
labels:
|
||||
kyuubi-role: KYUUBI_SERVER
|
||||
spec:
|
||||
containers:
|
||||
- name: kyuubi-server
|
||||
# TODO: replace this with the official repo
|
||||
image: yaooqinn/kyuubi:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: KYUUBI_JAVA_OPTS
|
||||
value: -Dkyuubi.frontend.bind.host=0.0.0.0
|
||||
ports:
|
||||
- name: frontend-port
|
||||
containerPort: 10009
|
||||
hostPort: 10009
|
||||
protocol: TCP
|
||||
@ -360,7 +360,8 @@ trait JDBCTests extends BasicJDBCTests {
|
||||
}
|
||||
}
|
||||
|
||||
test("kyuubi defined function - system_user") {
|
||||
// dockerfile use kyuubi as user which is not same with non-k8s env.
|
||||
ignore("kyuubi defined function - system_user") {
|
||||
withJdbcStatement() { statement =>
|
||||
val rs = statement.executeQuery("SELECT system_user()")
|
||||
assert(rs.next())
|
||||
|
||||
15
pom.xml
15
pom.xml
@ -126,6 +126,8 @@
|
||||
<slf4j.version>1.7.30</slf4j.version>
|
||||
<zookeeper.version>3.4.14</zookeeper.version>
|
||||
|
||||
<kubernetes-client.version>5.5.0</kubernetes-client.version>
|
||||
|
||||
<scalatest.version>3.2.9</scalatest.version>
|
||||
<iceberg.name>iceberg-spark3-runtime</iceberg.name>
|
||||
<iceberg.version>0.11.1</iceberg.version>
|
||||
@ -423,6 +425,12 @@
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.fabric8</groupId>
|
||||
<artifactId>kubernetes-client</artifactId>
|
||||
<version>${kubernetes-client.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-common</artifactId>
|
||||
@ -1599,5 +1607,12 @@
|
||||
<module>tools/spark-block-cleaner</module>
|
||||
</modules>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>kubernetes</id>
|
||||
<modules>
|
||||
<module>kubernetes/integration-tests</module>
|
||||
</modules>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user