From 3646ae08bd064ac2aa53ba10a411b067d046e098 Mon Sep 17 00:00:00 2001 From: Xianxun Ye Date: Wed, 25 Oct 2023 16:27:13 +0800 Subject: [PATCH] [KYUUBI #5405] [FLINK] Support Flink 1.18 ### _Why are the changes needed?_ Resolve: #5405 Support the Flink 1.18 ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request ### _Was this patch authored or co-authored using generative AI tooling?_ No Closes #5465 from YesOrNo828/flink-1.18. Closes #5405 a0010ca14 [Xianxun Ye] [KYUUBI #5405] [FLINK] Remove flink1.18 rc repo 2a4ae365c [Xianxun Ye] Update .github/workflows/master.yml d4d458dc7 [Xianxun Ye] [KYUUBI #5405] [FLINK] Update the flink1.18-rc3 repo 99172e3da [Xianxun Ye] [KYUUBI #5405] [FLINK] Using the staging repo during the RC stage 4c0cf887b [Xianxun Ye] [KYUUBI #5405] [FLINK] Using the staging repo during the RC stage c74f5c31b [Xianxun Ye] [KYUUBI #5405] [FLINK] fixed Pan's comments. 1933ebadd [Xianxun Ye] [KYUUBI #5405] [FLINK] Support Flink 1.18 Authored-by: Xianxun Ye Signed-off-by: Cheng Pan --- .github/workflows/master.yml | 5 +++++ .../org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala | 4 ++-- pom.xml | 7 +++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index c934c2d5e..1819c4850 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -183,6 +183,7 @@ jobs: flink: - '1.16' - '1.17' + - '1.18' flink-archive: [ "" ] comment: [ "normal" ] include: @@ -190,6 +191,10 @@ jobs: flink: '1.17' flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.16.1 -Dflink.archive.name=flink-1.16.1-bin-scala_2.12.tgz' comment: 'verify-on-flink-1.16-binary' + - java: 8 + flink: '1.17' + flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.18.0 -Dflink.archive.name=flink-1.18.0-bin-scala_2.12.tgz' + comment: 'verify-on-flink-1.18-binary' steps: - uses: actions/checkout@v3 - name: Tune Runner VM diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala index 7d42aae8c..06165272d 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala @@ -48,7 +48,7 @@ object FlinkEngineUtils extends Logging { val EMBEDDED_MODE_CLIENT_OPTIONS: Options = getEmbeddedModeClientOptions(new Options) - private def SUPPORTED_FLINK_VERSIONS = Set("1.16", "1.17").map(SemanticVersion.apply) + private def SUPPORTED_FLINK_VERSIONS = Set("1.16", "1.17", "1.18").map(SemanticVersion.apply) val FLINK_RUNTIME_VERSION: SemanticVersion = SemanticVersion(EnvironmentInformation.getVersion) @@ -119,7 +119,7 @@ object FlinkEngineUtils extends Logging { .build() .newInstance(flinkConf, commandLines) .asInstanceOf[DefaultContext] - } else if (FLINK_RUNTIME_VERSION === "1.17") { + } else if (FLINK_RUNTIME_VERSION >= "1.17") { invokeAs[DefaultContext]( classOf[DefaultContext], "load", diff --git a/pom.xml b/pom.xml index 059bc214b..f03770b9d 100644 --- a/pom.xml +++ b/pom.xml @@ -2304,6 +2304,13 @@ + + flink-1.18 + + 1.18.0 + + + zookeeper-3.6