diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index e8c982ca7..0c6ed989b 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -111,11 +111,12 @@ jobs: - '1.15' flink-archive: [ "" ] comment: [ "normal" ] - include: - - java: 8 - flink: '1.15' - flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.15.1 -Dflink.archive.name=flink-1.15.1-bin-scala_2.12.tgz' - comment: 'verify-flink-1.15' + # FIXME: Cross Flink versions verification is not supported yet + # include: + # - java: 8 + # flink: '1.15' + # flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.14.5 -Dflink.archive.name=flink-1.14.5-bin-scala_2.12.tgz' + # comment: 'verify-flink-1.14' steps: - uses: actions/checkout@v2 - name: Tune Runner VM diff --git a/docs/quick_start/quick_start.md b/docs/quick_start/quick_start.md index e0dd99b1b..2099ed352 100644 --- a/docs/quick_start/quick_start.md +++ b/docs/quick_start/quick_start.md @@ -154,7 +154,7 @@ Similar to `JAVA_HOME`, you can also set `FLINK_HOME` in different ways. However For example, ```bash -FLINK_HOME=/Downloads/flink-1.14.4 +FLINK_HOME=/Downloads/flink-1.15.1 ``` #### Setup Kyuubi Flink Configration diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala index 211105dea..79f23c7c6 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala @@ -22,8 +22,9 @@ import java.util import scala.collection.JavaConverters._ import scala.collection.immutable.ListMap +import scala.util.matching.Regex -import org.apache.kyuubi.{FLINK_COMPILE_VERSION, KyuubiException, KyuubiFunSuite, SCALA_COMPILE_VERSION} +import org.apache.kyuubi.{KyuubiException, KyuubiFunSuite} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.{ENGINE_FLINK_EXTRA_CLASSPATH, ENGINE_FLINK_JAVA_OPTIONS, ENGINE_FLINK_MEMORY} import org.apache.kyuubi.engine.flink.FlinkProcessBuilder._ @@ -45,16 +46,18 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { (FLINK_HADOOP_CLASSPATH_KEY -> s"${File.separator}hadoop") private def confStr: String = { conf.clone.set("yarn.tags", "KYUUBI").getAll - .map { case (k, v) => s"\\\n\t--conf $k=$v" } + .map { case (k, v) => s"\\\\\\n\\t--conf $k=$v" } .mkString(" ") } - private def compareActualAndExpected(builder: FlinkProcessBuilder): Unit = { + private def matchActualAndExpected(builder: FlinkProcessBuilder): Unit = { val actualCommands = builder.toString val classpathStr = constructClasspathStr(builder) val expectedCommands = s"$javaPath -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005 " + - s"-cp $classpathStr $mainClassStr \\\n\t--conf kyuubi.session.user=vinoyang $confStr" - assert(actualCommands == expectedCommands) + s"-cp $classpathStr $mainClassStr \\\\\\n\\t--conf kyuubi.session.user=vinoyang $confStr" + val regex = new Regex(expectedCommands) + val matcher = regex.pattern.matcher(actualCommands) + assert(matcher.matches()) } private def constructClasspathStr(builder: FlinkProcessBuilder) = { @@ -62,8 +65,8 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { builder.mainResource.foreach(classpathEntries.add) val flinkHome = builder.flinkHome - classpathEntries.add(s"$flinkHome$flinkSqlClientJarPathSuffix") - classpathEntries.add(s"$flinkHome$flinkLibPathSuffix") + classpathEntries.add(s"$flinkHome$flinkSqlClientJarPathSuffixRegex") + classpathEntries.add(s"$flinkHome$flinkLibPathSuffixRegex") classpathEntries.add(s"$flinkHome$flinkConfPathSuffix") val envMap = builder.env envMap.foreach { case (k, v) => @@ -78,9 +81,9 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { } private val javaPath = s"${envDefault("JAVA_HOME")}${File.separator}bin${File.separator}java" - private val flinkSqlClientJarPathSuffix = s"${File.separator}opt${File.separator}" + - s"flink-sql-client_$SCALA_COMPILE_VERSION-$FLINK_COMPILE_VERSION.jar" - private val flinkLibPathSuffix = s"${File.separator}lib${File.separator}*" + private val flinkSqlClientJarPathSuffixRegex = s"${File.separator}opt${File.separator}" + + s"flink-sql-client-.*.jar" + private val flinkLibPathSuffixRegex = s"${File.separator}lib${File.separator}\\*" private val flinkConfPathSuffix = s"${File.separator}conf" private val mainClassStr = "org.apache.kyuubi.engine.flink.FlinkSQLEngine" @@ -88,7 +91,7 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { val builder = new FlinkProcessBuilder("vinoyang", conf) { override def env: Map[String, String] = envWithAllHadoop } - compareActualAndExpected(builder) + matchActualAndExpected(builder) } test("all hadoop related environment variables are configured except FLINK_HADOOP_CLASSPATH") { @@ -102,6 +105,6 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite { override def env: Map[String, String] = envDefault + (FLINK_HADOOP_CLASSPATH_KEY -> s"${File.separator}hadoop") } - compareActualAndExpected(builder) + matchActualAndExpected(builder) } } diff --git a/pom.xml b/pom.xml index a90d36c49..0f8639a7d 100644 --- a/pom.xml +++ b/pom.xml @@ -115,8 +115,8 @@ 2.0.0 2.4.4 0.9.3 - 1.14.5 - _${scala.binary.version} + 1.15.1 + flink-${flink.version}-bin-scala_${scala.binary.version}.tgz ${apache.archive.dist}/flink/flink-${flink.version} false