From 86964fefbe794c8553cfa0d567a3cf3e43ef4a96 Mon Sep 17 00:00:00 2001 From: Luning Wang Date: Thu, 11 Aug 2022 15:31:59 +0800 Subject: [PATCH] [KYUUBI #3206] Change Flink default version to 1.15 ### _Why are the changes needed?_ ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #3207 from deadwind4/flink-default-version. Closes #3206 4c9b5d6c [Luning Wang] Fix test case to fit flink 1.14 and 1.15 09c854e2 [Luning Wang] change sql-client.jar name to fit flink 1.15 eef06d1e [Luning Wang] change the suffix to fit 1.15 93fb44f4 [Luning Wang] Comment out cross validation 8ea10218 [Luning Wang] Fix cross validation a5e5cdb9 [Luning Wang] Fix github CI Flink version 1ee8a5d9 [Luning Wang] [KYUUBI #3206] Change Flink default version to 1.15 Authored-by: Luning Wang Signed-off-by: Cheng Pan --- .github/workflows/master.yml | 11 ++++---- docs/quick_start/quick_start.md | 2 +- .../flink/FlinkProcessBuilderSuite.scala | 27 ++++++++++--------- pom.xml | 4 +-- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index e8c982ca7..0c6ed989b 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -111,11 +111,12 @@ jobs: - '1.15' flink-archive: [ "" ] comment: [ "normal" ] - include: - - java: 8 - flink: '1.15' - flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.15.1 -Dflink.archive.name=flink-1.15.1-bin-scala_2.12.tgz' - comment: 'verify-flink-1.15' + # FIXME: Cross Flink versions verification is not supported yet + # include: + # - java: 8 + # flink: '1.15' + # flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.14.5 -Dflink.archive.name=flink-1.14.5-bin-scala_2.12.tgz' + # comment: 'verify-flink-1.14' steps: - uses: actions/checkout@v2 - name: Tune Runner VM diff --git a/docs/quick_start/quick_start.md b/docs/quick_start/quick_start.md index e0dd99b1b..2099ed352 100644 --- a/docs/quick_start/quick_start.md +++ b/docs/quick_start/quick_start.md @@ -154,7 +154,7 @@ Similar to `JAVA_HOME`, you can also set `FLINK_HOME` in different ways. However For example, ```bash -FLINK_HOME=/Downloads/flink-1.14.4 +FLINK_HOME=/Downloads/flink-1.15.1 ``` #### Setup Kyuubi Flink Configration diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala index 211105dea..79f23c7c6 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala @@ -22,8 +22,9 @@ import java.util import scala.collection.JavaConverters._ import scala.collection.immutable.ListMap +import scala.util.matching.Regex -import org.apache.kyuubi.{FLINK_COMPILE_VERSION, KyuubiException, KyuubiFunSuite, SCALA_COMPILE_VERSION} +import org.apache.kyuubi.{KyuubiException, KyuubiFunSuite} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.{ENGINE_FLINK_EXTRA_CLASSPATH, ENGINE_FLINK_JAVA_OPTIONS, ENGINE_FLINK_MEMORY} import org.apache.kyuubi.engine.flink.FlinkProcessBuilder._ @@ -45,16 +46,18 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { (FLINK_HADOOP_CLASSPATH_KEY -> s"${File.separator}hadoop") private def confStr: String = { conf.clone.set("yarn.tags", "KYUUBI").getAll - .map { case (k, v) => s"\\\n\t--conf $k=$v" } + .map { case (k, v) => s"\\\\\\n\\t--conf $k=$v" } .mkString(" ") } - private def compareActualAndExpected(builder: FlinkProcessBuilder): Unit = { + private def matchActualAndExpected(builder: FlinkProcessBuilder): Unit = { val actualCommands = builder.toString val classpathStr = constructClasspathStr(builder) val expectedCommands = s"$javaPath -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005 " + - s"-cp $classpathStr $mainClassStr \\\n\t--conf kyuubi.session.user=vinoyang $confStr" - assert(actualCommands == expectedCommands) + s"-cp $classpathStr $mainClassStr \\\\\\n\\t--conf kyuubi.session.user=vinoyang $confStr" + val regex = new Regex(expectedCommands) + val matcher = regex.pattern.matcher(actualCommands) + assert(matcher.matches()) } private def constructClasspathStr(builder: FlinkProcessBuilder) = { @@ -62,8 +65,8 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { builder.mainResource.foreach(classpathEntries.add) val flinkHome = builder.flinkHome - classpathEntries.add(s"$flinkHome$flinkSqlClientJarPathSuffix") - classpathEntries.add(s"$flinkHome$flinkLibPathSuffix") + classpathEntries.add(s"$flinkHome$flinkSqlClientJarPathSuffixRegex") + classpathEntries.add(s"$flinkHome$flinkLibPathSuffixRegex") classpathEntries.add(s"$flinkHome$flinkConfPathSuffix") val envMap = builder.env envMap.foreach { case (k, v) => @@ -78,9 +81,9 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { } private val javaPath = s"${envDefault("JAVA_HOME")}${File.separator}bin${File.separator}java" - private val flinkSqlClientJarPathSuffix = s"${File.separator}opt${File.separator}" + - s"flink-sql-client_$SCALA_COMPILE_VERSION-$FLINK_COMPILE_VERSION.jar" - private val flinkLibPathSuffix = s"${File.separator}lib${File.separator}*" + private val flinkSqlClientJarPathSuffixRegex = s"${File.separator}opt${File.separator}" + + s"flink-sql-client-.*.jar" + private val flinkLibPathSuffixRegex = s"${File.separator}lib${File.separator}\\*" private val flinkConfPathSuffix = s"${File.separator}conf" private val mainClassStr = "org.apache.kyuubi.engine.flink.FlinkSQLEngine" @@ -88,7 +91,7 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { val builder = new FlinkProcessBuilder("vinoyang", conf) { override def env: Map[String, String] = envWithAllHadoop } - compareActualAndExpected(builder) + matchActualAndExpected(builder) } test("all hadoop related environment variables are configured except FLINK_HADOOP_CLASSPATH") { @@ -102,6 +105,6 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { override def env: Map[String, String] = envDefault + (FLINK_HADOOP_CLASSPATH_KEY -> s"${File.separator}hadoop") } - compareActualAndExpected(builder) + matchActualAndExpected(builder) } } diff --git a/pom.xml b/pom.xml index a90d36c49..0f8639a7d 100644 --- a/pom.xml +++ b/pom.xml @@ -115,8 +115,8 @@ 2.0.0 2.4.4 0.9.3 - 1.14.5 - _${scala.binary.version} + 1.15.1 + flink-${flink.version}-bin-scala_${scala.binary.version}.tgz ${apache.archive.dist}/flink/flink-${flink.version} false