[KYUUBI #3206] Change Flink default version to 1.15
### _Why are the changes needed?_ ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #3207 from deadwind4/flink-default-version. Closes #3206 4c9b5d6c [Luning Wang] Fix test case to fit flink 1.14 and 1.15 09c854e2 [Luning Wang] change sql-client.jar name to fit flink 1.15 eef06d1e [Luning Wang] change the suffix to fit 1.15 93fb44f4 [Luning Wang] Comment out cross validation 8ea10218 [Luning Wang] Fix cross validation a5e5cdb9 [Luning Wang] Fix github CI Flink version 1ee8a5d9 [Luning Wang] [KYUUBI #3206] Change Flink default version to 1.15 Authored-by: Luning Wang <wang4luning@gmail.com> Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Luning Wang
Cheng Pan
parent
b3723392db
commit
86964fefbe
11
.github/workflows/master.yml
vendored
11
.github/workflows/master.yml
vendored
@ -111,11 +111,12 @@ jobs:
|
||||
- '1.15'
|
||||
flink-archive: [ "" ]
|
||||
comment: [ "normal" ]
|
||||
include:
|
||||
- java: 8
|
||||
flink: '1.15'
|
||||
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.15.1 -Dflink.archive.name=flink-1.15.1-bin-scala_2.12.tgz'
|
||||
comment: 'verify-flink-1.15'
|
||||
# FIXME: Cross Flink versions verification is not supported yet
|
||||
# include:
|
||||
# - java: 8
|
||||
# flink: '1.15'
|
||||
# flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.14.5 -Dflink.archive.name=flink-1.14.5-bin-scala_2.12.tgz'
|
||||
# comment: 'verify-flink-1.14'
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Tune Runner VM
|
||||
|
||||
@ -154,7 +154,7 @@ Similar to `JAVA_HOME`, you can also set `FLINK_HOME` in different ways. However
|
||||
For example,
|
||||
|
||||
```bash
|
||||
FLINK_HOME=/Downloads/flink-1.14.4
|
||||
FLINK_HOME=/Downloads/flink-1.15.1
|
||||
```
|
||||
|
||||
#### Setup Kyuubi Flink Configration
|
||||
|
||||
@ -22,8 +22,9 @@ import java.util
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.immutable.ListMap
|
||||
import scala.util.matching.Regex
|
||||
|
||||
import org.apache.kyuubi.{FLINK_COMPILE_VERSION, KyuubiException, KyuubiFunSuite, SCALA_COMPILE_VERSION}
|
||||
import org.apache.kyuubi.{KyuubiException, KyuubiFunSuite}
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_FLINK_EXTRA_CLASSPATH, ENGINE_FLINK_JAVA_OPTIONS, ENGINE_FLINK_MEMORY}
|
||||
import org.apache.kyuubi.engine.flink.FlinkProcessBuilder._
|
||||
@ -45,16 +46,18 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite {
|
||||
(FLINK_HADOOP_CLASSPATH_KEY -> s"${File.separator}hadoop")
|
||||
private def confStr: String = {
|
||||
conf.clone.set("yarn.tags", "KYUUBI").getAll
|
||||
.map { case (k, v) => s"\\\n\t--conf $k=$v" }
|
||||
.map { case (k, v) => s"\\\\\\n\\t--conf $k=$v" }
|
||||
.mkString(" ")
|
||||
}
|
||||
private def compareActualAndExpected(builder: FlinkProcessBuilder): Unit = {
|
||||
private def matchActualAndExpected(builder: FlinkProcessBuilder): Unit = {
|
||||
val actualCommands = builder.toString
|
||||
val classpathStr = constructClasspathStr(builder)
|
||||
val expectedCommands =
|
||||
s"$javaPath -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005 " +
|
||||
s"-cp $classpathStr $mainClassStr \\\n\t--conf kyuubi.session.user=vinoyang $confStr"
|
||||
assert(actualCommands == expectedCommands)
|
||||
s"-cp $classpathStr $mainClassStr \\\\\\n\\t--conf kyuubi.session.user=vinoyang $confStr"
|
||||
val regex = new Regex(expectedCommands)
|
||||
val matcher = regex.pattern.matcher(actualCommands)
|
||||
assert(matcher.matches())
|
||||
}
|
||||
|
||||
private def constructClasspathStr(builder: FlinkProcessBuilder) = {
|
||||
@ -62,8 +65,8 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite {
|
||||
builder.mainResource.foreach(classpathEntries.add)
|
||||
|
||||
val flinkHome = builder.flinkHome
|
||||
classpathEntries.add(s"$flinkHome$flinkSqlClientJarPathSuffix")
|
||||
classpathEntries.add(s"$flinkHome$flinkLibPathSuffix")
|
||||
classpathEntries.add(s"$flinkHome$flinkSqlClientJarPathSuffixRegex")
|
||||
classpathEntries.add(s"$flinkHome$flinkLibPathSuffixRegex")
|
||||
classpathEntries.add(s"$flinkHome$flinkConfPathSuffix")
|
||||
val envMap = builder.env
|
||||
envMap.foreach { case (k, v) =>
|
||||
@ -78,9 +81,9 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite {
|
||||
}
|
||||
|
||||
private val javaPath = s"${envDefault("JAVA_HOME")}${File.separator}bin${File.separator}java"
|
||||
private val flinkSqlClientJarPathSuffix = s"${File.separator}opt${File.separator}" +
|
||||
s"flink-sql-client_$SCALA_COMPILE_VERSION-$FLINK_COMPILE_VERSION.jar"
|
||||
private val flinkLibPathSuffix = s"${File.separator}lib${File.separator}*"
|
||||
private val flinkSqlClientJarPathSuffixRegex = s"${File.separator}opt${File.separator}" +
|
||||
s"flink-sql-client-.*.jar"
|
||||
private val flinkLibPathSuffixRegex = s"${File.separator}lib${File.separator}\\*"
|
||||
private val flinkConfPathSuffix = s"${File.separator}conf"
|
||||
private val mainClassStr = "org.apache.kyuubi.engine.flink.FlinkSQLEngine"
|
||||
|
||||
@ -88,7 +91,7 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite {
|
||||
val builder = new FlinkProcessBuilder("vinoyang", conf) {
|
||||
override def env: Map[String, String] = envWithAllHadoop
|
||||
}
|
||||
compareActualAndExpected(builder)
|
||||
matchActualAndExpected(builder)
|
||||
}
|
||||
|
||||
test("all hadoop related environment variables are configured except FLINK_HADOOP_CLASSPATH") {
|
||||
@ -102,6 +105,6 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite {
|
||||
override def env: Map[String, String] = envDefault +
|
||||
(FLINK_HADOOP_CLASSPATH_KEY -> s"${File.separator}hadoop")
|
||||
}
|
||||
compareActualAndExpected(builder)
|
||||
matchActualAndExpected(builder)
|
||||
}
|
||||
}
|
||||
|
||||
4
pom.xml
4
pom.xml
@ -115,8 +115,8 @@
|
||||
<delta.version>2.0.0</delta.version>
|
||||
<failsafe.verion>2.4.4</failsafe.verion>
|
||||
<fb303.version>0.9.3</fb303.version>
|
||||
<flink.version>1.14.5</flink.version>
|
||||
<flink.module.scala.suffix>_${scala.binary.version}</flink.module.scala.suffix>
|
||||
<flink.version>1.15.1</flink.version>
|
||||
<flink.module.scala.suffix></flink.module.scala.suffix>
|
||||
<flink.archive.name>flink-${flink.version}-bin-scala_${scala.binary.version}.tgz</flink.archive.name>
|
||||
<flink.archive.mirror>${apache.archive.dist}/flink/flink-${flink.version}</flink.archive.mirror>
|
||||
<flink.archive.download.skip>false</flink.archive.download.skip>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user