From d63eed274905fe90d2bcdd607a56a0e70cc7936a Mon Sep 17 00:00:00 2001
From: yanghua <yanghua1127@gmail.com>
Date: Tue, 11 Jan 2022 12:45:41 +0800
Subject: [PATCH] =?UTF-8?q?[KYUUBI=20#1709]=20Introduce=20a=20IT=20module?=
 =?UTF-8?q?=20for=20Kyuubi=20and=20FlinkSQL=20Engine=20t=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…o decouple with kyuubi server

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #1711 from yanghua/KYUUBI-1709.

Closes #1709

95466130 [yanghua] refactor code
89f9c572 [yanghua] Reorder module list in root pom file
b664a4bb [yanghua] Rename IT module names
6b7fa88f [yanghua] Rename IT module names
b133e8e9 [yanghua] Reformatted pom indent
38444416 [yanghua] [KYUUBI #1709] Introduce a IT module for Kyuubi and FlinkSQL Engine to decouple with kyuubi server

Authored-by: yanghua <yanghua1127@gmail.com>
Signed-off-by: Cheng Pan <chengpan@apache.org>
---
 integration-tests/kyuubi-flink-it/pom.xml     | 91 +++++++++++++++++++
 .../WithKyuubiServerAndFlinkMiniCluster.scala | 28 ++++--
 .../operation}/FlinkOperationSuite.scala      |  6 +-
 integration-tests/pom.xml                     | 38 ++++++++
 kyuubi-server/pom.xml                         | 16 ++++
 pom.xml                                       |  1 +
 6 files changed, 169 insertions(+), 11 deletions(-)
 create mode 100644 integration-tests/kyuubi-flink-it/pom.xml
 rename kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndFlinkLocalCluster.scala => integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndFlinkMiniCluster.scala (51%)
 rename {kyuubi-server/src/test/scala/org/apache/kyuubi/operation/flink => integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation}/FlinkOperationSuite.scala (90%)
 create mode 100644 integration-tests/pom.xml

diff --git a/integration-tests/kyuubi-flink-it/pom.xml b/integration-tests/kyuubi-flink-it/pom.xml
new file mode 100644
index 000000000..971204e84
--- /dev/null
+++ b/integration-tests/kyuubi-flink-it/pom.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>integration-tests</artifactId>
+        <groupId>org.apache.kyuubi</groupId>
+        <version>1.5.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>kyuubi-flink-it_2.12</artifactId>
+    <name>Kyuubi Project IT Flink SQL</name>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-common_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-server_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-server_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-hive-jdbc-shaded</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Flink -->
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-runtime</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-table-runtime_${scala.binary.version}</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>jul-to-slf4j</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndFlinkLocalCluster.scala b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndFlinkMiniCluster.scala
similarity index 51%
rename from kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndFlinkLocalCluster.scala
rename to integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndFlinkMiniCluster.scala
index a6d5d92ed..0e0f28265 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndFlinkLocalCluster.scala
+++ b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndFlinkMiniCluster.scala
@@ -15,24 +15,36 @@
  * limitations under the License.
  */
 
-package org.apache.kyuubi
+package org.apache.kyuubi.it.flink
 
-import scala.sys.process._
+import org.apache.flink.configuration.{Configuration, RestOptions}
+import org.apache.flink.runtime.minicluster.{MiniCluster, MiniClusterConfiguration}
 
-import org.apache.kyuubi.engine.flink.FlinkProcessBuilder
+import org.apache.kyuubi.WithKyuubiServer
 
-trait WithKyuubiServerAndFlinkLocalCluster extends WithKyuubiServer {
+trait WithKyuubiServerAndFlinkMiniCluster extends WithKyuubiServer {
 
-  private lazy val FLINK_HOME: String =
-    new FlinkProcessBuilder(Utils.currentUser, conf).FLINK_HOME
+  protected lazy val flinkConfig = new Configuration()
+  protected var miniCluster: MiniCluster = _
 
   override def beforeAll(): Unit = {
-    s"$FLINK_HOME/bin/start-cluster.sh".!
+    startMiniCluster()
     super.beforeAll()
   }
 
   override def afterAll(): Unit = {
     super.afterAll()
-    s"$FLINK_HOME/bin/stop-cluster.sh".!
+    miniCluster.close()
+  }
+
+  private def startMiniCluster(): Unit = {
+    val cfg = new MiniClusterConfiguration.Builder()
+      .setConfiguration(flinkConfig)
+      .setNumSlotsPerTaskManager(1)
+      .build
+    miniCluster = new MiniCluster(cfg)
+    miniCluster.start()
+    flinkConfig.setString(RestOptions.ADDRESS, miniCluster.getRestAddress.get().getHost)
+    flinkConfig.setInteger(RestOptions.PORT, miniCluster.getRestAddress.get().getPort)
   }
 }
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/flink/FlinkOperationSuite.scala b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuite.scala
similarity index 90%
rename from kyuubi-server/src/test/scala/org/apache/kyuubi/operation/flink/FlinkOperationSuite.scala
rename to integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuite.scala
index 8915c1978..2ab44a4ef 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/flink/FlinkOperationSuite.scala
+++ b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuite.scala
@@ -15,15 +15,15 @@
  * limitations under the License.
  */
 
-package org.apache.kyuubi.operation.flink
+package org.apache.kyuubi.it.flink.operation
 
-import org.apache.kyuubi.WithKyuubiServerAndFlinkLocalCluster
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.config.KyuubiConf.{ENGINE_TYPE, FRONTEND_THRIFT_BINARY_BIND_PORT}
+import org.apache.kyuubi.it.flink.WithKyuubiServerAndFlinkMiniCluster
 import org.apache.kyuubi.operation.HiveJDBCTestHelper
 import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.TABLE_CAT
 
-class FlinkOperationSuite extends WithKyuubiServerAndFlinkLocalCluster with HiveJDBCTestHelper {
+class FlinkOperationSuite extends WithKyuubiServerAndFlinkMiniCluster with HiveJDBCTestHelper {
   override val conf: KyuubiConf = KyuubiConf()
     .set(ENGINE_TYPE, "FLINK_SQL")
     .set(FRONTEND_THRIFT_BINARY_BIND_PORT, 10019)
diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
new file mode 100644
index 000000000..df58e73d9
--- /dev/null
+++ b/integration-tests/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>kyuubi-parent</artifactId>
+        <groupId>org.apache.kyuubi</groupId>
+        <version>1.5.0-SNAPSHOT</version>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <packaging>pom</packaging>
+
+    <artifactId>integration-tests</artifactId>
+    <name>Kyuubi Integration Tests</name>
+
+    <modules>
+        <module>kyuubi-flink-it</module>
+    </modules>
+
+</project>
diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml
index 100875402..309f73bd8 100644
--- a/kyuubi-server/pom.xml
+++ b/kyuubi-server/pom.xml
@@ -460,5 +460,21 @@
     <build>
         <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
         <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>prepare-test-jar</id>
+                        <phase>test-compile</phase>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
     </build>
 </project>
diff --git a/pom.xml b/pom.xml
index dfa0df629..225fa5295 100644
--- a/pom.xml
+++ b/pom.xml
@@ -70,6 +70,7 @@
         <module>externals/kyuubi-flink-sql-engine</module>
         <module>externals/kyuubi-spark-sql-engine</module>
         <module>externals/kyuubi-trino-engine</module>
+        <module>integration-tests</module>
         <module>kyuubi-assembly</module>
         <module>kyuubi-common</module>
         <module>kyuubi-ctl</module>