[KYUUBI #7153] Share JAAS configuration for Zookeeper client to avoid server OOM

### Why are the changes needed?

Sharing jaas configuration for zookeeper client with same keytab and principal to avoid server oom due to nested jaas configuration.

fix issue https://github.com/apache/kyuubi/issues/7153

### How was this patch tested?

ut

### Was this patch authored or co-authored using generative AI tooling?

no

Closes #7154 from Z1Wu/fix/comm_reuse_zk_jass.

Closes #7153

3b0169a00 [Cheng Pan] Update kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala
5873d12f3 [Cheng Pan] Update kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala
0d8a18a4e [wuziyi] nit
ffa7d29fc [wuziyi] [fix] share jaas configuration for zookeeper client with same keytab and principal to avoid server oom due to recursive jaas configuration.

Lead-authored-by: wuziyi <wuziyi02@corp.netease.com>
Co-authored-by: Cheng Pan <pan3793@gmail.com>
Signed-off-by: Cheng Pan <chengpan@apache.org>
(cherry picked from commit f7e10e65d3)
Signed-off-by: Cheng Pan <chengpan@apache.org>

.github/actions/cache-engine-archives/action.yaml

@@ -27,4 +27,4 @@ runs:
         key: engine-archives
     - name: Show cached engine archives
       shell: bash
-      run: ls -lh /tmp/engine-archives
+      run: ls -lh /tmp/engine-archives || echo ''

.github/workflows/license.yml

@@ -41,10 +41,10 @@ jobs:
           distribution: temurin
           java-version: 8
           cache: 'maven'
-      - run: >-
-          build/mvn org.apache.rat:apache-rat-plugin:check
-          -Ptpcds -Pkubernetes-it
-          -Pspark-3.3 -Pspark-3.4 -Pspark-3.5
+      - run: |
+          build/mvn org.apache.rat:apache-rat-plugin:check \
+            -Ptpcds -Pkubernetes-it \
+            -Pspark-3.3 -Pspark-3.4 -Pspark-3.5 -Pspark-4.0
       - name: Upload rat report
         if: failure()
         uses: actions/upload-artifact@v4

.github/workflows/master.yml

@@ -77,7 +77,7 @@ jobs:
             comment: 'verify-on-spark-3.4-binary'
           - java: 17
             spark: '3.5'
-            spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.0-preview2 -Dspark.archive.name=spark-4.0.0-preview2-bin-hadoop3.tgz'
+            spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.0 -Dspark.archive.name=spark-4.0.0-bin-hadoop3.tgz'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-4.0-binary'
     env:

.github/workflows/style.yml

@@ -132,9 +132,3 @@ jobs:
         with:
           name: super-linter-log
           path: super-linter.log
-      - name: check bin directory
-        uses: ludeeus/action-shellcheck@1.1.0
-        with:
-          # TODO: enable for all folders step by step
-          scandir: './bin'
-          severity: error

dev/dependencyList

@@ -30,7 +30,7 @@ arrow-vector/16.0.0//arrow-vector-16.0.0.jar
 checker-qual/3.42.0//checker-qual-3.42.0.jar
 classgraph/4.8.138//classgraph-4.8.138.jar
 commons-codec/1.17.1//commons-codec-1.17.1.jar
-commons-lang3/3.17.0//commons-lang3-3.17.0.jar
+commons-lang3/3.18.0//commons-lang3-3.18.0.jar
 error_prone_annotations/2.23.0//error_prone_annotations-2.23.0.jar
 failsafe/3.3.2//failsafe-3.3.2.jar
 failureaccess/1.0.2//failureaccess-1.0.2.jar
@@ -97,7 +97,7 @@ jetty-util-ajax/9.4.57.v20241219//jetty-util-ajax-9.4.57.v20241219.jar
 jetty-util/9.4.57.v20241219//jetty-util-9.4.57.v20241219.jar
 jline/2.14.6//jline-2.14.6.jar
 jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
-kafka-clients/3.5.2//kafka-clients-3.5.2.jar
+kafka-clients/3.9.1//kafka-clients-3.9.1.jar
 kubernetes-client-api/6.13.1//kubernetes-client-api-6.13.1.jar
 kubernetes-client/6.13.1//kubernetes-client-6.13.1.jar
 kubernetes-httpclient-okhttp/6.13.1//kubernetes-httpclient-okhttp-6.13.1.jar

extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/RebalanceBeforeWriting.scala

@@ -45,15 +45,6 @@ trait RepartitionBuilderWithRebalance extends RepartitionBuilder {
       }
     }
   }
-
-  override def canInsertRepartitionByExpression(plan: LogicalPlan): Boolean = {
-    super.canInsertRepartitionByExpression(plan) && {
-      plan match {
-        case _: RebalancePartitions => false
-        case _ => true
-      }
-    }
-  }
 }
 
 /**

extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/RepartitionBeforeWritingBase.scala

@@ -118,6 +118,7 @@ trait RepartitionBeforeWriteHelper extends Rule[LogicalPlan] {
       case _: Window => true
       case s: Sort if s.global => true
       case _: RepartitionOperation => true
+      case _: RebalancePartitions => true
       case _: GlobalLimit => true
       case _ => false
     }.isDefined
@@ -131,8 +132,8 @@ trait RepartitionBeforeWriteHelper extends Rule[LogicalPlan] {
       case SubqueryAlias(_, child) => canInsert(child)
       case Limit(_, _) => false
       case _: Sort => false
-      case _: RepartitionByExpression => false
-      case _: Repartition => false
+      case _: RepartitionOperation => false
+      case _: RebalancePartitions => false
       case _ => true
     }
 

extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWriting.scala

@@ -50,12 +50,12 @@ trait InsertZorderHelper extends Rule[LogicalPlan] with ZorderBuilder {
 
   def canInsertZorder(query: LogicalPlan): Boolean = query match {
     case Project(_, child) => canInsertZorder(child)
-    case _: RepartitionByExpression | _: Repartition
+    case _: RepartitionOperation | _: RebalancePartitions
         if !conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED) => true
     // TODO: actually, we can force zorder even if existed some shuffle
     case _: Sort => false
-    case _: RepartitionByExpression => false
-    case _: Repartition => false
+    case _: RepartitionOperation => false
+    case _: RebalancePartitions => false
     case _ => true
   }
 

extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/RebalanceBeforeWritingSuite.scala

@@ -28,18 +28,21 @@ import org.apache.kyuubi.sql.KyuubiSQLConf
 class RebalanceBeforeWritingSuite extends KyuubiSparkSQLExtensionTest {
 
   test("check rebalance exists") {
-    def check(df: => DataFrame, expectedRebalanceNum: Int = 1): Unit = {
+    def check(
+        df: => DataFrame,
+        expectedRebalanceNumEnabled: Int = 1,
+        expectedRebalanceNumDisabled: Int = 0): Unit = {
       withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") {
         assert(
           df.queryExecution.analyzed.collect {
             case r: RebalancePartitions => r
-          }.size == expectedRebalanceNum)
+          }.size == expectedRebalanceNumEnabled)
       }
       withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "false") {
         assert(
           df.queryExecution.analyzed.collect {
             case r: RebalancePartitions => r
-          }.isEmpty)
+          }.size == expectedRebalanceNumDisabled)
       }
     }
 
@@ -69,6 +72,14 @@ class RebalanceBeforeWritingSuite extends KyuubiSparkSQLExtensionTest {
           check(sql("INSERT INTO TABLE tmp1 SELECT * FROM VALUES(1),(2),(3) AS t(c1)"))
         }
 
+        withTable("tmp1") {
+          sql(s"CREATE TABLE tmp1 (c1 int) $storage")
+          check(
+            sql("INSERT INTO TABLE tmp1 SELECT /*+ REBALANCE */ * FROM VALUES(1),(2),(3) AS t(c1)"),
+            1,
+            1)
+        }
+
         withTable("tmp1", "tmp2") {
           sql(s"CREATE TABLE tmp1 (c1 int) $storage")
           sql(s"CREATE TABLE tmp2 (c1 int) $storage")

extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/RebalanceBeforeWriting.scala

@@ -45,15 +45,6 @@ trait RepartitionBuilderWithRebalance extends RepartitionBuilder {
       }
     }
   }
-
-  override def canInsertRepartitionByExpression(plan: LogicalPlan): Boolean = {
-    super.canInsertRepartitionByExpression(plan) && {
-      plan match {
-        case _: RebalancePartitions => false
-        case _ => true
-      }
-    }
-  }
 }
 
 /**

extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/RepartitionBeforeWritingBase.scala

@@ -99,6 +99,7 @@ trait RepartitionBeforeWriteHelper extends Rule[LogicalPlan] {
       case _: Window => true
       case s: Sort if s.global => true
       case _: RepartitionOperation => true
+      case _: RebalancePartitions => true
       case _: GlobalLimit => true
       case _ => false
     }.isDefined
@@ -112,8 +113,8 @@ trait RepartitionBeforeWriteHelper extends Rule[LogicalPlan] {
       case SubqueryAlias(_, child) => canInsert(child)
       case Limit(_, _) => false
       case _: Sort => false
-      case _: RepartitionByExpression => false
-      case _: Repartition => false
+      case _: RepartitionOperation => false
+      case _: RebalancePartitions => false
       case _ => true
     }
 

extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWriting.scala

@@ -49,12 +49,12 @@ trait InsertZorderHelper extends Rule[LogicalPlan] with ZorderBuilder {
 
   def canInsertZorder(query: LogicalPlan): Boolean = query match {
     case Project(_, child) => canInsertZorder(child)
-    case _: RepartitionByExpression | _: Repartition
+    case _: RepartitionOperation | _: RebalancePartitions
         if !conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED) => true
     // TODO: actually, we can force zorder even if existed some shuffle
     case _: Sort => false
-    case _: RepartitionByExpression => false
-    case _: Repartition => false
+    case _: RepartitionOperation => false
+    case _: RebalancePartitions => false
     case _ => true
   }
 

extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/RebalanceBeforeWritingSuite.scala

@@ -29,19 +29,22 @@ import org.apache.kyuubi.sql.KyuubiSQLConf
 class RebalanceBeforeWritingSuite extends KyuubiSparkSQLExtensionTest {
 
   test("check rebalance exists") {
-    def check(df: => DataFrame, expectedRebalanceNum: Int = 1): Unit = {
+    def check(
+        df: => DataFrame,
+        expectedRebalanceNumEnabled: Int = 1,
+        expectedRebalanceNumDisabled: Int = 0): Unit = {
       withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") {
         withListener(df) { write =>
           assert(write.collect {
             case r: RebalancePartitions => r
-          }.size == expectedRebalanceNum)
+          }.size == expectedRebalanceNumEnabled)
         }
       }
       withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "false") {
         withListener(df) { write =>
           assert(write.collect {
             case r: RebalancePartitions => r
-          }.isEmpty)
+          }.size == expectedRebalanceNumDisabled)
         }
       }
     }
@@ -72,6 +75,14 @@ class RebalanceBeforeWritingSuite extends KyuubiSparkSQLExtensionTest {
           check(sql("INSERT INTO TABLE tmp1 SELECT * FROM VALUES(1),(2),(3) AS t(c1)"))
         }
 
+        withTable("tmp1") {
+          sql(s"CREATE TABLE tmp1 (c1 int) $storage")
+          check(
+            sql("INSERT INTO TABLE tmp1 SELECT /*+ REBALANCE */ * FROM VALUES(1),(2),(3) AS t(c1)"),
+            1,
+            1)
+        }
+
         withTable("tmp1", "tmp2") {
           sql(s"CREATE TABLE tmp1 (c1 int) $storage")
           sql(s"CREATE TABLE tmp2 (c1 int) $storage")

extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/RebalanceBeforeWriting.scala

@@ -45,15 +45,6 @@ trait RepartitionBuilderWithRebalance extends RepartitionBuilder {
       }
     }
   }
-
-  override def canInsertRepartitionByExpression(plan: LogicalPlan): Boolean = {
-    super.canInsertRepartitionByExpression(plan) && {
-      plan match {
-        case _: RebalancePartitions => false
-        case _ => true
-      }
-    }
-  }
 }
 
 /**

extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/RepartitionBeforeWritingBase.scala

@@ -99,6 +99,7 @@ trait RepartitionBeforeWriteHelper extends Rule[LogicalPlan] {
       case _: Window => true
       case s: Sort if s.global => true
       case _: RepartitionOperation => true
+      case _: RebalancePartitions => true
       case _: GlobalLimit => true
       case _ => false
     }.isDefined
@@ -112,8 +113,8 @@ trait RepartitionBeforeWriteHelper extends Rule[LogicalPlan] {
       case SubqueryAlias(_, child) => canInsert(child)
       case Limit(_, _) => false
       case _: Sort => false
-      case _: RepartitionByExpression => false
-      case _: Repartition => false
+      case _: RepartitionOperation => false
+      case _: RebalancePartitions => false
       case _ => true
     }
 

extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWriting.scala

@@ -49,12 +49,12 @@ trait InsertZorderHelper extends Rule[LogicalPlan] with ZorderBuilder {
 
   def canInsertZorder(query: LogicalPlan): Boolean = query match {
     case Project(_, child) => canInsertZorder(child)
-    case _: RepartitionByExpression | _: Repartition
+    case _: RepartitionOperation | _: RebalancePartitions
         if !conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED) => true
     // TODO: actually, we can force zorder even if existed some shuffle
     case _: Sort => false
-    case _: RepartitionByExpression => false
-    case _: Repartition => false
+    case _: RepartitionOperation => false
+    case _: RebalancePartitions => false
     case _ => true
   }
 

extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/RebalanceBeforeWritingSuite.scala

@@ -29,19 +29,22 @@ import org.apache.kyuubi.sql.KyuubiSQLConf
 class RebalanceBeforeWritingSuite extends KyuubiSparkSQLExtensionTest {
 
   test("check rebalance exists") {
-    def check(df: => DataFrame, expectedRebalanceNum: Int = 1): Unit = {
+    def check(
+        df: => DataFrame,
+        expectedRebalanceNumEnabled: Int = 1,
+        expectedRebalanceNumDisabled: Int = 0): Unit = {
       withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") {
         withListener(df) { write =>
           assert(write.collect {
             case r: RebalancePartitions => r
-          }.size == expectedRebalanceNum)
+          }.size == expectedRebalanceNumEnabled)
         }
       }
       withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "false") {
         withListener(df) { write =>
           assert(write.collect {
             case r: RebalancePartitions => r
-          }.isEmpty)
+          }.size == expectedRebalanceNumDisabled)
         }
       }
     }
@@ -72,6 +75,14 @@ class RebalanceBeforeWritingSuite extends KyuubiSparkSQLExtensionTest {
           check(sql("INSERT INTO TABLE tmp1 SELECT * FROM VALUES(1),(2),(3) AS t(c1)"))
         }
 
+        withTable("tmp1") {
+          sql(s"CREATE TABLE tmp1 (c1 int) $storage")
+          check(
+            sql("INSERT INTO TABLE tmp1 SELECT /*+ REBALANCE */ * FROM VALUES(1),(2),(3) AS t(c1)"),
+            1,
+            1)
+        }
+
         withTable("tmp1", "tmp2") {
           sql(s"CREATE TABLE tmp1 (c1 int) $storage")
           sql(s"CREATE TABLE tmp2 (c1 int) $storage")

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala

@@ -222,4 +222,8 @@ class SparkSQLSessionManager private (name: String, spark: SparkSession)
       opHandle: OperationHandle): Path = {
     new Path(getSessionResultSavePath(sessionHandle), opHandle.identifier.toString)
   }
+
+  override private[kyuubi] def isEngineContextStopped = {
+    spark.sparkContext.isStopped
+  }
 }

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala

@@ -24,6 +24,7 @@ import org.apache.spark.sql.{AnalysisException, SparkSession}
 import org.apache.spark.ui.SparkUIUtils.formatDuration
 
 import org.apache.kyuubi.KyuubiSQLException
+import org.apache.kyuubi.config.KyuubiConf.SESSION_IDLE_TIMEOUT
 import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY
 import org.apache.kyuubi.engine.spark.events.SessionEvent
 import org.apache.kyuubi.engine.spark.operation.SparkSQLOperationManager
@@ -57,6 +58,13 @@ class SparkSessionImpl(
     }
   }
 
+  override val sessionIdleTimeoutThreshold: Long = {
+    conf.get(SESSION_IDLE_TIMEOUT.key)
+      .map(_.toLong)
+      .getOrElse(
+        sessionManager.getConf.get(SESSION_IDLE_TIMEOUT))
+  }
+
   private val sessionEvent = SessionEvent(this)
 
   override def open(): Unit = {

kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala

@@ -345,10 +345,15 @@ abstract class SessionManager(name: String) extends CompositeService(name) {
     if (idleTimeout > 0) {
       val checkTask = new Runnable {
         override def run(): Unit = {
-          if (!shutdown && System.currentTimeMillis() - latestLogoutTime > idleTimeout &&
-            getActiveUserSessionCount <= 0) {
-            info(s"Idled for more than $idleTimeout ms, terminating")
-            stop()
+          if (!shutdown) {
+            if (System.currentTimeMillis() - latestLogoutTime > idleTimeout &&
+              getActiveUserSessionCount <= 0) {
+              info(s"Idled for more than $idleTimeout ms, terminating")
+              stop()
+            } else if (isEngineContextStopped) {
+              error(s"Engine's SparkContext is stopped, terminating")
+              stop()
+            }
           }
         }
       }
@@ -360,4 +365,6 @@ abstract class SessionManager(name: String) extends CompositeService(name) {
         TimeUnit.MILLISECONDS)
     }
   }
+
+  private[kyuubi] def isEngineContextStopped: Boolean = false
 }

kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala

@@ -19,6 +19,7 @@ package org.apache.kyuubi.ha.client.zookeeper
 
 import java.io.{File, IOException}
 import java.nio.charset.StandardCharsets
+import java.util.concurrent.ConcurrentHashMap
 import javax.security.auth.login.Configuration
 
 import scala.util.Random
@@ -38,6 +39,13 @@ import org.apache.kyuubi.util.reflect.DynConstructors
 
 object ZookeeperClientProvider extends Logging {
 
+  /**
+   * Share JAAS configuration for Zookeeper client with same keytab and principal to
+   * avoid server OOM due to each new JAAS configuration references the previous instance.
+   * See KYUUBI #7154 for more details.
+   */
+  val jaasConfigurationCache = new ConcurrentHashMap[(String, String), Configuration]()
+
   /**
    * Create a [[CuratorFramework]] instance to be used as the ZooKeeper client
    * Use the [[ZookeeperACLProvider]] to create appropriate ACLs
@@ -113,22 +121,26 @@ object ZookeeperClientProvider extends Logging {
           System.setProperty("zookeeper.server.principal", zkServerPrincipal)
         }
         val zkClientPrincipal = KyuubiHadoopUtils.getServerPrincipal(principal)
-        // HDFS-16591 makes breaking change on JaasConfiguration
-        val jaasConf = DynConstructors.builder()
-          .impl( // Hadoop 3.3.5 and above
-            "org.apache.hadoop.security.authentication.util.JaasConfiguration",
-            classOf[String],
-            classOf[String],
-            classOf[String])
-          .impl( // Hadoop 3.3.4 and previous
-            // scalastyle:off
-            "org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager$JaasConfiguration",
-            // scalastyle:on
-            classOf[String],
-            classOf[String],
-            classOf[String])
-          .build[Configuration]()
-          .newInstance("KyuubiZooKeeperClient", zkClientPrincipal, keytab)
+        val jaasConf = jaasConfigurationCache.computeIfAbsent(
+          (principal, keytab),
+          _ => {
+            // HDFS-16591 makes breaking change on JaasConfiguration
+            DynConstructors.builder()
+              .impl( // Hadoop 3.3.5 and above
+                "org.apache.hadoop.security.authentication.util.JaasConfiguration",
+                classOf[String],
+                classOf[String],
+                classOf[String])
+              .impl( // Hadoop 3.3.4 and previous
+                // scalastyle:off
+                "org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager$JaasConfiguration",
+                // scalastyle:on
+                classOf[String],
+                classOf[String],
+                classOf[String])
+              .build[Configuration]()
+              .newInstance("KyuubiZooKeeperClient", zkClientPrincipal, keytab)
+          })
         Configuration.setConfiguration(jaasConf)
       case _ =>
     }

kyuubi-server/web-ui/package-lock.json

@@ -1597,6 +1597,18 @@
         "node": ">=8"
       }
     },
+    "node_modules/call-bind-apply-helpers": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
+      "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "function-bind": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/callsites": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -1947,6 +1959,19 @@
         "node": ">=12"
       }
     },
+    "node_modules/dunder-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
+      "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.2.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/element-plus": {
       "version": "2.2.13",
       "resolved": "https://registry.npmjs.org/element-plus/-/element-plus-2.2.13.tgz",
@@ -1984,6 +2009,47 @@
         "url": "https://github.com/fb55/entities?sponsor=1"
       }
     },
+    "node_modules/es-define-property": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+      "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-errors": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+      "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-object-atoms": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
+      "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
+      "dependencies": {
+        "es-errors": "^1.3.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-set-tostringtag": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
+      "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.6",
+        "has-tostringtag": "^1.0.2",
+        "hasown": "^2.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/esbuild": {
       "version": "0.18.20",
       "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.18.20.tgz",
@@ -2536,12 +2602,14 @@
       }
     },
     "node_modules/form-data": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
+      "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
       "dependencies": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
+        "es-set-tostringtag": "^2.1.0",
+        "hasown": "^2.0.2",
         "mime-types": "^2.1.12"
       },
       "engines": {
@@ -2568,6 +2636,14 @@
         "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
       }
     },
+    "node_modules/function-bind": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+      "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/functional-red-black-tree": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
@@ -2583,6 +2659,41 @@
         "node": "*"
       }
     },
+    "node_modules/get-intrinsic": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
+      "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.2",
+        "es-define-property": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "es-object-atoms": "^1.1.1",
+        "function-bind": "^1.1.2",
+        "get-proto": "^1.0.1",
+        "gopd": "^1.2.0",
+        "has-symbols": "^1.1.0",
+        "hasown": "^2.0.2",
+        "math-intrinsics": "^1.1.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/get-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
+      "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
+      "dependencies": {
+        "dunder-proto": "^1.0.1",
+        "es-object-atoms": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/get-stdin": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz",
@@ -2661,6 +2772,17 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/gopd": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+      "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/grapheme-splitter": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz",
@@ -2676,6 +2798,42 @@
         "node": ">=8"
       }
     },
+    "node_modules/has-symbols": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
+      "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-tostringtag": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
+      "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
+      "dependencies": {
+        "has-symbols": "^1.0.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/hasown": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+      "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+      "dependencies": {
+        "function-bind": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/html-encoding-sniffer": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",
@@ -3089,6 +3247,14 @@
         "semver": "bin/semver.js"
       }
     },
+    "node_modules/math-intrinsics": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
+      "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/md5-hex": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/md5-hex/-/md5-hex-3.0.1.tgz",
@@ -5665,6 +5831,15 @@
       "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==",
       "dev": true
     },
+    "call-bind-apply-helpers": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
+      "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
+      "requires": {
+        "es-errors": "^1.3.0",
+        "function-bind": "^1.1.2"
+      }
+    },
     "callsites": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -5933,6 +6108,16 @@
         "webidl-conversions": "^7.0.0"
       }
     },
+    "dunder-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
+      "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
+      "requires": {
+        "call-bind-apply-helpers": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.2.0"
+      }
+    },
     "element-plus": {
       "version": "2.2.13",
       "resolved": "https://registry.npmjs.org/element-plus/-/element-plus-2.2.13.tgz",
@@ -5961,6 +6146,35 @@
       "integrity": "sha512-o4q/dYJlmyjP2zfnaWDUC6A3BQFmVTX+tZPezK7k0GLSU9QYCauscf5Y+qcEPzKL+EixVouYDgLQK5H9GrLpkg==",
       "dev": true
     },
+    "es-define-property": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+      "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="
+    },
+    "es-errors": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+      "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="
+    },
+    "es-object-atoms": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
+      "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
+      "requires": {
+        "es-errors": "^1.3.0"
+      }
+    },
+    "es-set-tostringtag": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
+      "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
+      "requires": {
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.6",
+        "has-tostringtag": "^1.0.2",
+        "hasown": "^2.0.2"
+      }
+    },
     "esbuild": {
       "version": "0.18.20",
       "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.18.20.tgz",
@@ -6366,12 +6580,14 @@
       "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA=="
     },
     "form-data": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
+      "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
       "requires": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
+        "es-set-tostringtag": "^2.1.0",
+        "hasown": "^2.0.2",
         "mime-types": "^2.1.12"
       }
     },
@@ -6388,6 +6604,11 @@
       "dev": true,
       "optional": true
     },
+    "function-bind": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+      "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="
+    },
     "functional-red-black-tree": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
@@ -6400,6 +6621,32 @@
       "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==",
       "dev": true
     },
+    "get-intrinsic": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
+      "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
+      "requires": {
+        "call-bind-apply-helpers": "^1.0.2",
+        "es-define-property": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "es-object-atoms": "^1.1.1",
+        "function-bind": "^1.1.2",
+        "get-proto": "^1.0.1",
+        "gopd": "^1.2.0",
+        "has-symbols": "^1.1.0",
+        "hasown": "^2.0.2",
+        "math-intrinsics": "^1.1.0"
+      }
+    },
+    "get-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
+      "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
+      "requires": {
+        "dunder-proto": "^1.0.1",
+        "es-object-atoms": "^1.0.0"
+      }
+    },
     "get-stdin": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz",
@@ -6451,6 +6698,11 @@
         "slash": "^3.0.0"
       }
     },
+    "gopd": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+      "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="
+    },
     "grapheme-splitter": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz",
@@ -6463,6 +6715,27 @@
       "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true
     },
+    "has-symbols": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
+      "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="
+    },
+    "has-tostringtag": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
+      "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
+      "requires": {
+        "has-symbols": "^1.0.3"
+      }
+    },
+    "hasown": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+      "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+      "requires": {
+        "function-bind": "^1.1.2"
+      }
+    },
     "html-encoding-sniffer": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",
@@ -6782,6 +7055,11 @@
         }
       }
     },
+    "math-intrinsics": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
+      "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="
+    },
     "md5-hex": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/md5-hex/-/md5-hex-3.0.1.tgz",

pom.xml

@@ -134,7 +134,7 @@
         <commons-codec.version>1.17.1</commons-codec.version>
         <commons-collections.version>3.2.2</commons-collections.version>
         <commons-io.version>2.16.1</commons-io.version>
-        <commons-lang3.version>3.17.0</commons-lang3.version>
+        <commons-lang3.version>3.18.0</commons-lang3.version>
         <delta.artifact>delta-spark_${scala.binary.version}</delta.artifact>
         <delta.version>3.2.0</delta.version>
         <failsafe.verion>3.3.2</failsafe.verion>
@@ -172,7 +172,7 @@
         <jetty.version>9.4.57.v20241219</jetty.version>
         <jline.version>2.14.6</jline.version>
         <junit.version>4.13.2</junit.version>
-        <kafka.version>3.5.2</kafka.version>
+        <kafka.version>3.9.1</kafka.version>
         <kubernetes-client.version>6.13.1</kubernetes-client.version>
         <kyuubi-relocated.version>0.4.1</kyuubi-relocated.version>
         <kyuubi-relocated-zookeeper.artifacts>kyuubi-relocated-zookeeper-34</kyuubi-relocated-zookeeper.artifacts>
@@ -2042,18 +2042,19 @@
                 <module>extensions/spark/kyuubi-spark-connector-hive</module>
             </modules>
             <properties>
-                <spark.version>4.0.0-preview2</spark.version>
+                <spark.version>4.0.0</spark.version>
                 <spark.binary.version>4.0</spark.binary.version>
                 <antlr4.version>4.13.1</antlr4.version>
-                <!-- TODO: update once Delta support Spark 4.0.0-preview2 -->
+                <!-- TODO: update once Delta support Spark 4.0 -->
                 <delta.version>4.0.0rc1</delta.version>
                 <delta.artifact>delta-spark_${scala.binary.version}</delta.artifact>
                 <!-- TODO: update once Hudi support Spark 4.0 -->
                 <hudi.artifact>hudi-spark3.5-bundle_${scala.binary.version}</hudi.artifact>
                 <!-- TODO: update once Iceberg support Spark 4.0 -->
                 <iceberg.artifact>iceberg-spark-runtime-3.5_${scala.binary.version}</iceberg.artifact>
-                <!-- TODO: update once Paimon support Spark 4.0 -->
-                <paimon.artifact>paimon-spark-3.5</paimon.artifact>
+                <!-- TODO: update once Paimon support Spark 4.0.
+                           paimon-spark-3.5 contains Scala 2.12 classes cause conflicts with Scala 2.13 -->
+                <paimon.artifact>paimon-common</paimon.artifact>
                 <maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest</maven.plugin.scalatest.exclude.tags>
                 <spark.archive.name>spark-${spark.version}-bin-hadoop3.tgz</spark.archive.name>
             </properties>