208ab3af62
### _Why are the changes needed?_ This PR aims to migrate the vanilla Zookeeper and Curator to the Kyuubi Shaded Zookeeper. It's the first step to adapting JDK 17. There is a known issue [ZOOKEEPER-3779](https://issues.apache.org/jira/browse/ZOOKEEPER-3779) that Zookeeper 3.4 client can not run on JDK 14 and above, in https://github.com/apache/kyuubi-shaded/pull/5, we fixed this issue by a surgical. With the above fixing, zk-3.4 and zk-3.6 clients both work well on JDK 17, we just randomly pick some cases to make sure zk-3.6 is tested zk-3.4 client supports zk-3.4+ server, but zk-3.6 client only supports zk-3.5+ server; in the meanwhile, zk-3.4 is adopted widely, (CDH 5/6, HDP, EMR created before 2023). We are sticky to zk-3.4 to ensure that Kyuubi can be out-of-box in the most existing Hadoop cluster but also provide zk-3.6 as an alternative(simply replace the kyuubi-shaded-zk-3.4 jar w/ kyuubi-shaded-zk-3.6, or build w/ -Pzookeeper-3.6) for users who concerns that zk-3.4 is EOL. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4852 from pan3793/shaded-zk. Closes #4852 d960cc945 [Cheng Pan] remove staging repo 1b3622080 [Cheng Pan] Switch to Kyuubi Shaded Zookeeper Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
421 lines
20 KiB
XML
421 lines
20 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-parent</artifactId>
|
|
<version>1.8.0-SNAPSHOT</version>
|
|
<relativePath>../../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<artifactId>kyuubi-spark-sql-engine_2.12</artifactId>
|
|
<packaging>jar</packaging>
|
|
<name>Kyuubi Project Engine Spark SQL</name>
|
|
<url>https://kyuubi.apache.org/</url>
|
|
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-events_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-ha_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>io.grpc</groupId>
|
|
<artifactId>grpc-core</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>io.grpc</groupId>
|
|
<artifactId>grpc-protobuf</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-repl_${scala.binary.version}</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-compiler</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-reflect</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-api</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<!--
|
|
Spark requires `commons-collections` and `commons-io` but got them from transitive
|
|
dependencies of `hadoop-client`. As we are using Hadoop Shaded Client, we need add
|
|
them explicitly. See more details at SPARK-33212.
|
|
-->
|
|
<dependency>
|
|
<groupId>commons-collections</groupId>
|
|
<artifactId>commons-collections</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>commons-io</groupId>
|
|
<artifactId>commons-io</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-hive_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>${hive.jdbc.artifact}</artifactId>
|
|
<version>${project.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-runtime</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.iceberg</groupId>
|
|
<artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>io.delta</groupId>
|
|
<artifactId>delta-core_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.kyuubi</groupId>
|
|
<artifactId>kyuubi-zookeeper_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>io.etcd</groupId>
|
|
<artifactId>jetcd-launcher</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.vladsch.flexmark</groupId>
|
|
<artifactId>flexmark-all</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-shade-plugin</artifactId>
|
|
<configuration>
|
|
<shadedArtifactAttached>false</shadedArtifactAttached>
|
|
<artifactSet>
|
|
<includes>
|
|
<include>com.google.android:annotations</include>
|
|
<include>com.google.api.grpc:proto-google-common-protos</include>
|
|
<include>com.google.code.gson:gson</include>
|
|
<include>com.google.errorprone:error_prone_annotations</include>
|
|
<include>com.google.guava:*</include>
|
|
<include>com.google.j2objc:j2objc-annotations</include>
|
|
<include>com.google.protobuf:*</include>
|
|
<include>io.etcd:*</include>
|
|
<include>io.grpc:*</include>
|
|
<include>io.netty:*</include>
|
|
<include>io.perfmark:perfmark-api</include>
|
|
<include>io.vertx:*</include>
|
|
<include>net.jodah:failsafe</include>
|
|
<include>org.apache.hive:hive-service-rpc</include>
|
|
<include>org.apache.kyuubi:*</include>
|
|
<include>org.apache.thrift:*</include>
|
|
<include>org.checkerframework:checker-qual</include>
|
|
<include>org.codehaus.mojo:animal-sniffer-annotations</include>
|
|
</includes>
|
|
</artifactSet>
|
|
<filters>
|
|
<filter>
|
|
<artifact>*:*</artifact>
|
|
<excludes>
|
|
<exclude>**/*.proto</exclude>
|
|
<exclude>META-INF/*.SF</exclude>
|
|
<exclude>META-INF/*.DSA</exclude>
|
|
<exclude>META-INF/*.RSA</exclude>
|
|
<exclude>META-INF/DEPENDENCIES</exclude>
|
|
<exclude>META-INF/LICENSE.txt</exclude>
|
|
<exclude>META-INF/NOTICE.txt</exclude>
|
|
<exclude>META-INF/maven/**</exclude>
|
|
<exclude>LICENSE.txt</exclude>
|
|
<exclude>NOTICE.txt</exclude>
|
|
<exclude>mozilla/**</exclude>
|
|
<exclude>**/module-info.class</exclude>
|
|
</excludes>
|
|
</filter>
|
|
</filters>
|
|
<relocations>
|
|
<relocation>
|
|
<pattern>org.apache.hive.service.rpc.thrift</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.org.apache.hive.service.rpc.thrift</shadedPattern>
|
|
<includes>
|
|
<include>org.apache.hive.service.rpc.thrift.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.facebook.fb303</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.facebook.fb303</shadedPattern>
|
|
<includes>
|
|
<include>com.facebook.fb303.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>org.apache.thrift</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.org.apache.thrift</shadedPattern>
|
|
<includes>
|
|
<include>org.apache.thrift.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>io.etcd</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.io.etcd</shadedPattern>
|
|
<includes>
|
|
<include>io.etcd.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>io.grpc</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.io.grpc</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>io.netty</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.io.netty</shadedPattern>
|
|
<includes>
|
|
<include>io.netty.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>io.perfmark</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.io.perfmark</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>io.vertx</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.io.vertx</shadedPattern>
|
|
<includes>
|
|
<include>io.vertx.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>net.jodah</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.net.jodah</shadedPattern>
|
|
<includes>
|
|
<include>net.jodah.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>android.annotation</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.android.annotation</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.common</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.common</shadedPattern>
|
|
<includes>
|
|
<include>com.google.common.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.thirdparty</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.thirdparty</shadedPattern>
|
|
<includes>
|
|
<include>com.google.thirdparty.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.protobuf</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.protobuf</shadedPattern>
|
|
<includes>
|
|
<include>com.google.protobuf.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>org.codehaus.mojo.animal_sniffer</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.org.codehaus.mojo.animal_sniffer</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.j2objc.annotations</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.j2objc.annotations</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.errorprone.annotations</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.errorprone.annotations</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>org.checkerframework</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.org.checkerframework</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.gson</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.gson</shadedPattern>
|
|
</relocation>
|
|
<!--
|
|
For `com.google.api.grpc:proto-google-common-protos`, do not directly define pattern
|
|
as `common.google`, otherwise, otherwise, the relocation result may be uncertain due
|
|
to the change of rule order.
|
|
-->
|
|
<relocation>
|
|
<pattern>com.google.api</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.api</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.cloud</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.cloud</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.geo</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.geo</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.logging</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.logging</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.longrunning</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.longrunning</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.rpc</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.rpc</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.type</pattern>
|
|
<shadedPattern>${kyuubi.shade.packageName}.com.google.type</shadedPattern>
|
|
</relocation>
|
|
</relocations>
|
|
<transformers>
|
|
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"></transformer>
|
|
</transformers>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<goals>
|
|
<goal>shade</goal>
|
|
</goals>
|
|
<phase>package</phase>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-antrun-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>rename-native-library</id>
|
|
<goals>
|
|
<goal>run</goal>
|
|
</goals>
|
|
<phase>package</phase>
|
|
<configuration>
|
|
<target>
|
|
<echo level="info">unpacking netty jar</echo>
|
|
<unzip dest="${project.build.directory}/unpacked/" src="${project.build.directory}/${project.artifactId}-${project.version}.jar"></unzip>
|
|
<echo level="info">renaming netty native libraries</echo>
|
|
<move includeemptydirs="false" todir="${project.build.directory}/unpacked/META-INF/native">
|
|
<fileset dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
|
|
<mapper from="libnetty_transport_native_epoll_(aarch_64.so|x86_64.so)" to="liborg_apache_kyuubi_shade_netty_transport_native_epoll_\1" type="regexp"></mapper>
|
|
</move>
|
|
<echo level="info">deleting META-INF/native-image folder</echo>
|
|
<delete dir="${project.build.directory}/unpacked/META-INF/native-image"></delete>
|
|
<echo level="info">repackaging netty jar</echo>
|
|
<jar basedir="${project.build.directory}/unpacked" destfile="${project.build.directory}/${project.artifactId}-${project.version}.jar"></jar>
|
|
</target>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-jar-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>prepare-test-jar</id>
|
|
<goals>
|
|
<goal>test-jar</goal>
|
|
</goals>
|
|
<phase>test-compile</phase>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
|
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
|
</build>
|
|
</project>
|