[KYUUBI #4852] Switch to Kyuubi Shaded Zookeeper
### _Why are the changes needed?_ This PR aims to migrate the vanilla Zookeeper and Curator to the Kyuubi Shaded Zookeeper. It's the first step to adapting JDK 17. There is a known issue [ZOOKEEPER-3779](https://issues.apache.org/jira/browse/ZOOKEEPER-3779) that Zookeeper 3.4 client can not run on JDK 14 and above, in https://github.com/apache/kyuubi-shaded/pull/5, we fixed this issue by a surgical. With the above fixing, zk-3.4 and zk-3.6 clients both work well on JDK 17, we just randomly pick some cases to make sure zk-3.6 is tested zk-3.4 client supports zk-3.4+ server, but zk-3.6 client only supports zk-3.5+ server; in the meanwhile, zk-3.4 is adopted widely, (CDH 5/6, HDP, EMR created before 2023). We are sticky to zk-3.4 to ensure that Kyuubi can be out-of-box in the most existing Hadoop cluster but also provide zk-3.6 as an alternative(simply replace the kyuubi-shaded-zk-3.4 jar w/ kyuubi-shaded-zk-3.6, or build w/ -Pzookeeper-3.6) for users who concerns that zk-3.4 is EOL. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4852 from pan3793/shaded-zk. Closes #4852 d960cc945 [Cheng Pan] remove staging repo 1b3622080 [Cheng Pan] Switch to Kyuubi Shaded Zookeeper Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Cheng Pan
parent
52464c25da
commit
208ab3af62
6
.github/workflows/master.yml
vendored
6
.github/workflows/master.yml
vendored
@ -57,17 +57,17 @@ jobs:
|
||||
include:
|
||||
- java: 8
|
||||
spark: '3.3'
|
||||
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz'
|
||||
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz -Pzookeeper-3.6'
|
||||
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
|
||||
comment: 'verify-on-spark-3.1-binary'
|
||||
- java: 8
|
||||
spark: '3.3'
|
||||
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz'
|
||||
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
|
||||
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
|
||||
comment: 'verify-on-spark-3.2-binary'
|
||||
- java: 8
|
||||
spark: '3.3'
|
||||
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.0 -Dspark.archive.name=spark-3.4.0-bin-hadoop3.tgz'
|
||||
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.0 -Dspark.archive.name=spark-3.4.0-bin-hadoop3.tgz -Pzookeeper-3.6'
|
||||
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
|
||||
comment: 'verify-on-spark-3.4-binary'
|
||||
env:
|
||||
|
||||
@ -212,9 +212,6 @@ com.google.android:annotations
|
||||
commons-lang:commons-lang
|
||||
commons-logging:commons-logging
|
||||
org.apache.commons:commons-lang3
|
||||
org.apache.curator:curator-client
|
||||
org.apache.curator:curator-framework
|
||||
org.apache.curator:curator-recipes
|
||||
org.apache.derby:derby
|
||||
com.google.errorprone:error_prone_annotations
|
||||
net.jodah:failsafe
|
||||
@ -318,7 +315,6 @@ io.swagger.core.v3:swagger-jaxrs2
|
||||
io.swagger.core.v3:swagger-models
|
||||
io.vertx:vertx-core
|
||||
io.vertx:vertx-grpc
|
||||
org.apache.zookeeper:zookeeper
|
||||
com.squareup.retrofit2:retrofit
|
||||
com.squareup.okhttp3:okhttp
|
||||
org.apache.kafka:kafka-clients
|
||||
|
||||
@ -92,15 +92,6 @@ Copyright 2001-2020 The Apache Software Foundation
|
||||
Apache Commons Logging
|
||||
Copyright 2003-2013 The Apache Software Foundation
|
||||
|
||||
Curator Client
|
||||
Copyright 2011-2017 The Apache Software Foundation
|
||||
|
||||
Curator Framework
|
||||
Copyright 2011-2017 The Apache Software Foundation
|
||||
|
||||
Curator Recipes
|
||||
Copyright 2011-2017 The Apache Software Foundation
|
||||
|
||||
=========================================================================
|
||||
== NOTICE file corresponding to section 4(d) of the Apache License,
|
||||
== Version 2.0, in this case for the Apache Derby distribution.
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
#
|
||||
|
||||
## Zookeeper Shell Client Entrance
|
||||
CLASS="org.apache.zookeeper.ZooKeeperMain"
|
||||
CLASS="org.apache.kyuubi.shaded.zookeeper.ZooKeeperMain"
|
||||
|
||||
export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"
|
||||
|
||||
|
||||
@ -32,9 +32,6 @@ commons-collections/3.2.2//commons-collections-3.2.2.jar
|
||||
commons-lang/2.6//commons-lang-2.6.jar
|
||||
commons-lang3/3.12.0//commons-lang3-3.12.0.jar
|
||||
commons-logging/1.1.3//commons-logging-1.1.3.jar
|
||||
curator-client/2.12.0//curator-client-2.12.0.jar
|
||||
curator-framework/2.12.0//curator-framework-2.12.0.jar
|
||||
curator-recipes/2.12.0//curator-recipes-2.12.0.jar
|
||||
derby/10.14.2.0//derby-10.14.2.0.jar
|
||||
error_prone_annotations/2.14.0//error_prone_annotations-2.14.0.jar
|
||||
failsafe/2.4.4//failsafe-2.4.4.jar
|
||||
@ -195,5 +192,4 @@ units/1.6//units-1.6.jar
|
||||
vertx-core/4.3.2//vertx-core-4.3.2.jar
|
||||
vertx-grpc/4.3.2//vertx-grpc-4.3.2.jar
|
||||
zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
|
||||
zookeeper/3.4.14//zookeeper-3.4.14.jar
|
||||
zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar
|
||||
|
||||
28
externals/kyuubi-flink-sql-engine/pom.xml
vendored
28
externals/kyuubi-flink-sql-engine/pom.xml
vendored
@ -186,20 +186,15 @@
|
||||
<shadedArtifactAttached>false</shadedArtifactAttached>
|
||||
<artifactSet>
|
||||
<includes>
|
||||
<include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
|
||||
<include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
|
||||
<include>com.fasterxml.jackson.core:*</include>
|
||||
<include>com.fasterxml.jackson.module:*</include>
|
||||
<include>com.google.guava:failureaccess</include>
|
||||
<include>com.google.guava:guava</include>
|
||||
<include>commons-codec:commons-codec</include>
|
||||
<include>org.apache.commons:commons-lang3</include>
|
||||
<include>org.apache.curator:curator-client</include>
|
||||
<include>org.apache.curator:curator-framework</include>
|
||||
<include>org.apache.curator:curator-recipes</include>
|
||||
<include>org.apache.hive:hive-service-rpc</include>
|
||||
<include>org.apache.thrift:*</include>
|
||||
<include>org.apache.zookeeper:*</include>
|
||||
<include>org.apache.kyuubi:*</include>
|
||||
</includes>
|
||||
</artifactSet>
|
||||
<filters>
|
||||
@ -228,13 +223,6 @@
|
||||
<include>com.fasterxml.jackson.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.curator</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
|
||||
<includes>
|
||||
<include>org.apache.curator.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>com.google.common</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.com.google.common</shadedPattern>
|
||||
@ -278,20 +266,6 @@
|
||||
<include>org.apache.thrift.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.jute</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.jute</shadedPattern>
|
||||
<includes>
|
||||
<include>org.apache.jute.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.zookeeper</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.zookeeper</shadedPattern>
|
||||
<includes>
|
||||
<include>org.apache.zookeeper.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
</relocations>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"></transformer>
|
||||
|
||||
16
externals/kyuubi-hive-sql-engine/pom.xml
vendored
16
externals/kyuubi-hive-sql-engine/pom.xml
vendored
@ -179,12 +179,7 @@
|
||||
<include>com.fasterxml.jackson.core:jackson-core</include>
|
||||
<include>com.fasterxml.jackson.core:jackson-databind</include>
|
||||
<include>com.fasterxml.jackson.module:jackson-module-scala_${scala.binary.version}</include>
|
||||
<include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
|
||||
<include>org.apache.kyuubi:kyuubi-events_${scala.binary.version}</include>
|
||||
<include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
|
||||
<include>org.apache.curator:curator-client</include>
|
||||
<include>org.apache.curator:curator-framework</include>
|
||||
<include>org.apache.curator:curator-recipes</include>
|
||||
<include>org.apache.kyuubi:*</include>
|
||||
</includes>
|
||||
</artifactSet>
|
||||
<filters>
|
||||
@ -205,15 +200,6 @@
|
||||
</excludes>
|
||||
</filter>
|
||||
</filters>
|
||||
<relocations>
|
||||
<relocation>
|
||||
<pattern>org.apache.curator</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
|
||||
<includes>
|
||||
<include>org.apache.curator.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
</relocations>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
|
||||
29
externals/kyuubi-spark-sql-engine/pom.xml
vendored
29
externals/kyuubi-spark-sql-engine/pom.xml
vendored
@ -194,15 +194,9 @@
|
||||
<include>io.perfmark:perfmark-api</include>
|
||||
<include>io.vertx:*</include>
|
||||
<include>net.jodah:failsafe</include>
|
||||
<include>org.apache.curator:curator-client</include>
|
||||
<include>org.apache.curator:curator-framework</include>
|
||||
<include>org.apache.curator:curator-recipes</include>
|
||||
<include>org.apache.hive:hive-service-rpc</include>
|
||||
<include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
|
||||
<include>org.apache.kyuubi:kyuubi-events_${scala.binary.version}</include>
|
||||
<include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
|
||||
<include>org.apache.kyuubi:*</include>
|
||||
<include>org.apache.thrift:*</include>
|
||||
<include>org.apache.zookeeper:zookeeper</include>
|
||||
<include>org.checkerframework:checker-qual</include>
|
||||
<include>org.codehaus.mojo:animal-sniffer-annotations</include>
|
||||
</includes>
|
||||
@ -227,27 +221,6 @@
|
||||
</filter>
|
||||
</filters>
|
||||
<relocations>
|
||||
<relocation>
|
||||
<pattern>org.apache.curator</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
|
||||
<includes>
|
||||
<include>org.apache.curator.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.zookeeper</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.zookeeper</shadedPattern>
|
||||
<includes>
|
||||
<include>org.apache.zookeeper.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.jute</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.jute</shadedPattern>
|
||||
<includes>
|
||||
<include>org.apache.jute.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.hive.service.rpc.thrift</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.hive.service.rpc.thrift</shadedPattern>
|
||||
|
||||
@ -68,6 +68,11 @@
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client-api</artifactId>
|
||||
@ -78,21 +83,6 @@
|
||||
<artifactId>hadoop-client-runtime</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-framework</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-recipes</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>jsr305</artifactId>
|
||||
|
||||
@ -48,6 +48,11 @@
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client-api</artifactId>
|
||||
@ -60,16 +65,6 @@
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-framework</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-recipes</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.github.scopt</groupId>
|
||||
<artifactId>scopt_${scala.binary.version}</artifactId>
|
||||
@ -86,11 +81,6 @@
|
||||
<version>${snakeyaml.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
|
||||
|
||||
@ -38,18 +38,8 @@
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-framework</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-recipes</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
|
||||
package org.apache.kyuubi.ha.client
|
||||
|
||||
import org.apache.curator.utils.ZKPaths
|
||||
import org.apache.kyuubi.shaded.curator.utils.ZKPaths
|
||||
|
||||
object DiscoveryPaths {
|
||||
def makePath(parent: String, firstChild: String, restChildren: String*): String = {
|
||||
|
||||
@ -17,13 +17,12 @@
|
||||
|
||||
package org.apache.kyuubi.ha.client.zookeeper
|
||||
|
||||
import org.apache.curator.framework.api.ACLProvider
|
||||
import org.apache.zookeeper.ZooDefs
|
||||
import org.apache.zookeeper.data.ACL
|
||||
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
import org.apache.kyuubi.ha.HighAvailabilityConf
|
||||
import org.apache.kyuubi.ha.client.AuthTypes
|
||||
import org.apache.kyuubi.shaded.curator.framework.api.ACLProvider
|
||||
import org.apache.kyuubi.shaded.zookeeper.ZooDefs
|
||||
import org.apache.kyuubi.shaded.zookeeper.data.ACL
|
||||
|
||||
class ZookeeperACLProvider(conf: KyuubiConf) extends ACLProvider {
|
||||
|
||||
|
||||
@ -24,8 +24,6 @@ import javax.security.auth.login.Configuration
|
||||
import scala.util.Random
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting
|
||||
import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
|
||||
import org.apache.curator.retry._
|
||||
import org.apache.hadoop.security.UserGroupInformation
|
||||
|
||||
import org.apache.kyuubi.Logging
|
||||
@ -34,6 +32,8 @@ import org.apache.kyuubi.ha.HighAvailabilityConf._
|
||||
import org.apache.kyuubi.ha.client.{AuthTypes, RetryPolicies}
|
||||
import org.apache.kyuubi.ha.client.RetryPolicies._
|
||||
import org.apache.kyuubi.reflection.DynConstructors
|
||||
import org.apache.kyuubi.shaded.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
|
||||
import org.apache.kyuubi.shaded.curator.retry._
|
||||
import org.apache.kyuubi.util.KyuubiHadoopUtils
|
||||
|
||||
object ZookeeperClientProvider extends Logging {
|
||||
|
||||
@ -25,23 +25,6 @@ import java.util.concurrent.atomic.AtomicBoolean
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting
|
||||
import org.apache.curator.framework.CuratorFramework
|
||||
import org.apache.curator.framework.recipes.atomic.{AtomicValue, DistributedAtomicInteger}
|
||||
import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex
|
||||
import org.apache.curator.framework.recipes.nodes.PersistentNode
|
||||
import org.apache.curator.framework.state.ConnectionState
|
||||
import org.apache.curator.framework.state.ConnectionState.CONNECTED
|
||||
import org.apache.curator.framework.state.ConnectionState.LOST
|
||||
import org.apache.curator.framework.state.ConnectionState.RECONNECTED
|
||||
import org.apache.curator.framework.state.ConnectionStateListener
|
||||
import org.apache.curator.retry.RetryForever
|
||||
import org.apache.curator.utils.ZKPaths
|
||||
import org.apache.zookeeper.CreateMode
|
||||
import org.apache.zookeeper.CreateMode.PERSISTENT
|
||||
import org.apache.zookeeper.KeeperException
|
||||
import org.apache.zookeeper.KeeperException.NodeExistsException
|
||||
import org.apache.zookeeper.WatchedEvent
|
||||
import org.apache.zookeeper.Watcher
|
||||
|
||||
import org.apache.kyuubi.KYUUBI_VERSION
|
||||
import org.apache.kyuubi.KyuubiException
|
||||
@ -58,6 +41,23 @@ import org.apache.kyuubi.ha.client.ServiceNodeInfo
|
||||
import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider.buildZookeeperClient
|
||||
import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider.getGracefulStopThreadDelay
|
||||
import org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient.connectionChecker
|
||||
import org.apache.kyuubi.shaded.curator.framework.CuratorFramework
|
||||
import org.apache.kyuubi.shaded.curator.framework.recipes.atomic.{AtomicValue, DistributedAtomicInteger}
|
||||
import org.apache.kyuubi.shaded.curator.framework.recipes.locks.InterProcessSemaphoreMutex
|
||||
import org.apache.kyuubi.shaded.curator.framework.recipes.nodes.PersistentNode
|
||||
import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState
|
||||
import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState.CONNECTED
|
||||
import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState.LOST
|
||||
import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState.RECONNECTED
|
||||
import org.apache.kyuubi.shaded.curator.framework.state.ConnectionStateListener
|
||||
import org.apache.kyuubi.shaded.curator.retry.RetryForever
|
||||
import org.apache.kyuubi.shaded.curator.utils.ZKPaths
|
||||
import org.apache.kyuubi.shaded.zookeeper.CreateMode
|
||||
import org.apache.kyuubi.shaded.zookeeper.CreateMode.PERSISTENT
|
||||
import org.apache.kyuubi.shaded.zookeeper.KeeperException
|
||||
import org.apache.kyuubi.shaded.zookeeper.KeeperException.NodeExistsException
|
||||
import org.apache.kyuubi.shaded.zookeeper.WatchedEvent
|
||||
import org.apache.kyuubi.shaded.zookeeper.Watcher
|
||||
import org.apache.kyuubi.util.ThreadUtils
|
||||
|
||||
class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient {
|
||||
|
||||
@ -25,11 +25,7 @@ import javax.security.auth.login.Configuration
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
import org.apache.curator.framework.CuratorFrameworkFactory
|
||||
import org.apache.curator.retry.ExponentialBackoffRetry
|
||||
import org.apache.hadoop.util.StringUtils
|
||||
import org.apache.zookeeper.ZooDefs
|
||||
import org.apache.zookeeper.data.ACL
|
||||
import org.scalatest.time.SpanSugar._
|
||||
|
||||
import org.apache.kyuubi.{KerberizedTestHelper, KYUUBI_VERSION}
|
||||
@ -39,6 +35,10 @@ import org.apache.kyuubi.ha.client._
|
||||
import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient
|
||||
import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider._
|
||||
import org.apache.kyuubi.service._
|
||||
import org.apache.kyuubi.shaded.curator.framework.CuratorFrameworkFactory
|
||||
import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry
|
||||
import org.apache.kyuubi.shaded.zookeeper.ZooDefs
|
||||
import org.apache.kyuubi.shaded.zookeeper.data.ACL
|
||||
import org.apache.kyuubi.zookeeper.EmbeddedZookeeper
|
||||
import org.apache.kyuubi.zookeeper.ZookeeperConf.ZK_CLIENT_PORT
|
||||
|
||||
|
||||
@ -108,10 +108,6 @@
|
||||
<pattern>org.apache.commons</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.commons</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.curator</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.hive</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.hive</shadedPattern>
|
||||
@ -120,18 +116,10 @@
|
||||
<pattern>org.apache.http</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.http</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.jute</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.jute</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.thrift</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.thrift</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.zookeeper</pattern>
|
||||
<shadedPattern>${kyuubi.shade.packageName}.org.apache.zookeeper</shadedPattern>
|
||||
</relocation>
|
||||
</relocations>
|
||||
</configuration>
|
||||
<executions>
|
||||
|
||||
@ -102,24 +102,14 @@
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-framework</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@ -25,9 +25,9 @@ import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
import org.apache.curator.framework.CuratorFrameworkFactory;
|
||||
import org.apache.curator.retry.ExponentialBackoffRetry;
|
||||
import org.apache.kyuubi.shaded.curator.framework.CuratorFramework;
|
||||
import org.apache.kyuubi.shaded.curator.framework.CuratorFrameworkFactory;
|
||||
import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry;
|
||||
|
||||
class ZooKeeperHiveClientHelper {
|
||||
// Pattern for key1=value1;key2=value2
|
||||
|
||||
@ -28,7 +28,6 @@ import io.swagger.v3.oas.annotations.media.{ArraySchema, Content, Schema}
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponse
|
||||
import io.swagger.v3.oas.annotations.tags.Tag
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.apache.zookeeper.KeeperException.NoNodeException
|
||||
|
||||
import org.apache.kyuubi.{KYUUBI_VERSION, Logging, Utils}
|
||||
import org.apache.kyuubi.client.api.v1.dto.{Engine, OperationData, ServerData, SessionData}
|
||||
@ -41,6 +40,7 @@ import org.apache.kyuubi.operation.{KyuubiOperation, OperationHandle}
|
||||
import org.apache.kyuubi.server.KyuubiServer
|
||||
import org.apache.kyuubi.server.api.{ApiRequestContext, ApiUtils}
|
||||
import org.apache.kyuubi.session.{KyuubiSession, SessionHandle}
|
||||
import org.apache.kyuubi.shaded.zookeeper.KeeperException.NoNodeException
|
||||
|
||||
@Tag(name = "Admin")
|
||||
@Produces(Array(MediaType.APPLICATION_JSON))
|
||||
|
||||
@ -37,8 +37,8 @@
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@ -48,24 +48,6 @@
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-framework</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-client</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
@ -20,11 +20,10 @@ package org.apache.kyuubi.zookeeper
|
||||
import java.io.File
|
||||
import java.net.InetSocketAddress
|
||||
|
||||
import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer}
|
||||
|
||||
import org.apache.kyuubi.Utils._
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
import org.apache.kyuubi.service.{AbstractService, ServiceState}
|
||||
import org.apache.kyuubi.shaded.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer}
|
||||
import org.apache.kyuubi.zookeeper.ZookeeperConf._
|
||||
|
||||
class EmbeddedZookeeper extends AbstractService("EmbeddedZookeeper") {
|
||||
|
||||
@ -17,12 +17,11 @@
|
||||
|
||||
package org.apache.kyuubi.zookeeper
|
||||
|
||||
import org.apache.curator.framework.CuratorFrameworkFactory
|
||||
import org.apache.curator.framework.imps.CuratorFrameworkState
|
||||
import org.apache.curator.retry.ExponentialBackoffRetry
|
||||
|
||||
import org.apache.kyuubi.KyuubiFunSuite
|
||||
import org.apache.kyuubi.config.KyuubiConf
|
||||
import org.apache.kyuubi.shaded.curator.framework.CuratorFrameworkFactory
|
||||
import org.apache.kyuubi.shaded.curator.framework.imps.CuratorFrameworkState
|
||||
import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry
|
||||
import org.apache.kyuubi.zookeeper.ZookeeperConf.{ZK_CLIENT_PORT, ZK_CLIENT_PORT_ADDRESS}
|
||||
|
||||
class EmbeddedZookeeperSuite extends KyuubiFunSuite {
|
||||
|
||||
76
pom.xml
76
pom.xml
@ -129,7 +129,6 @@
|
||||
<commons-io.version>2.11.0</commons-io.version>
|
||||
<commons-lang.version>2.6</commons-lang.version>
|
||||
<commons-lang3.version>3.12.0</commons-lang3.version>
|
||||
<curator.version>2.12.0</curator.version>
|
||||
<etcd.version>0.7.3</etcd.version>
|
||||
<delta.version>2.3.0</delta.version>
|
||||
<failsafe.verion>2.4.4</failsafe.verion>
|
||||
@ -168,6 +167,8 @@
|
||||
<kafka.version>3.4.0</kafka.version>
|
||||
<kubernetes-client.version>6.4.1</kubernetes-client.version>
|
||||
<kudu.version>1.15.0</kudu.version>
|
||||
<kyuubi-shaded-zookeeper.artifacts>kyuubi-shaded-zookeeper-34</kyuubi-shaded-zookeeper.artifacts>
|
||||
<kyuubi-shaded-zookeeper.version>0.1.0</kyuubi-shaded-zookeeper.version>
|
||||
<ldapsdk.version>6.0.5</ldapsdk.version>
|
||||
<log4j.version>2.20.0</log4j.version>
|
||||
<mysql.jdbc.version>8.0.32</mysql.jdbc.version>
|
||||
@ -203,7 +204,6 @@
|
||||
<trino.client.version>363</trino.client.version>
|
||||
<trino.tpcds.version>1.4</trino.tpcds.version>
|
||||
<trino.tpch.version>1.1</trino.tpch.version>
|
||||
<zookeeper.version>3.4.14</zookeeper.version>
|
||||
|
||||
<!-- webui -->
|
||||
<webui.skip>true</webui.skip>
|
||||
@ -260,6 +260,11 @@
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.kyuubi</groupId>
|
||||
<artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
|
||||
<version>${kyuubi-shaded-zookeeper.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-runtime</artifactId>
|
||||
@ -609,54 +614,6 @@
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!--
|
||||
Do not add hive dependencies here if they are dependencies of
|
||||
org.apache.spark:spark-hive_${scala.binary.version}.
|
||||
We should leave these dependencies' versions as they are defined in
|
||||
spark-parent_${scala.binary.version}-${spark.version}.pom.
|
||||
Otherwise, kyuubi-spark-sql-engine tests may encounter hive version compatibility issue.
|
||||
-->
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-framework</artifactId>
|
||||
<version>${curator.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-recipes</artifactId>
|
||||
<version>${curator.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-client</artifactId>
|
||||
<version>${curator.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
@ -1116,18 +1073,6 @@
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<version>${zookeeper.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>*</groupId>
|
||||
<artifactId>*</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.etcd</groupId>
|
||||
<artifactId>jetcd-core</artifactId>
|
||||
@ -2232,6 +2177,13 @@
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>zookeeper-3.6</id>
|
||||
<properties>
|
||||
<kyuubi-shaded-zookeeper.artifacts>kyuubi-shaded-zookeeper-36</kyuubi-shaded-zookeeper.artifacts>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark-provided</id>
|
||||
<properties>
|
||||
|
||||
@ -195,6 +195,14 @@ This file is divided into 3 sections:
|
||||
of Commons Lang 2 (package org.apache.commons.lang.*)</customMessage>
|
||||
</check>
|
||||
|
||||
<check customId="kyuubishaded" level="error" class="org.scalastyle.scalariform.TokenChecker" enabled="true">
|
||||
<parameters>
|
||||
<parameter name="regex">org\.apache\.curator\.</parameter>
|
||||
<parameter name="regex">org\.apache\.zookeeper\.</parameter>
|
||||
</parameters>
|
||||
<customMessage>Use Kyuubi shaded classes (package org.apache.kyuubi.shaded.*) instead classes</customMessage>
|
||||
</check>
|
||||
|
||||
<check customId="extractopt" level="error" class="org.scalastyle.scalariform.TokenChecker" enabled="true">
|
||||
<parameters><parameter name="regex">extractOpt</parameter></parameters>
|
||||
<customMessage>Use Utils.jsonOption(x).map(.extract[T]) instead of .extractOpt[T], as the latter
|
||||
|
||||
Loading…
Reference in New Issue
Block a user