apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #5113] Remove dummy Spark Kudu connector module

Browse Source 
### _Why are the changes needed?_

It was planned but actually delayed, remove this dummy module to save CI and avoid confusing users and release managers.

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request

Closes #5113 from pan3793/remove-kudu.

Closes #5113

ff8fd2e6a [Cheng Pan] Remove Spark Kudu connector

Authored-by: Cheng Pan <chengpan@apache.org>
Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Cheng Pan 2023-08-01 17:05:42 +08:00
parent ab3c333122
commit 94b2b9bb33
No known key found for this signature in database
GPG Key ID: 8001952629BCC75D
8 changed files with 1 additions and 419 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/style.yml vendored
View File

@ -67,7 +67,7 @@ jobs:
MVN_OPT="-DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip"
build/mvn clean install ${MVN_OPT} -Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.2,tpcds
build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-1 -Pspark-3.1
build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-kudu,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3
build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3
build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-4 -Pspark-3.4
- name: Scalastyle with maven

5
dev/kyuubi-codecov/pom.xml
View File

@ -197,11 +197,6 @@
<artifactId>kyuubi-extension-spark-3-3_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-spark-connector-kudu_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-spark-connector-hive_${scala.binary.version}</artifactId>

225
extensions/spark/kyuubi-spark-connector-kudu/pom.xml
View File

@ -1,225 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-parent</artifactId>
<version>1.8.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>kyuubi-spark-connector-kudu_2.12</artifactId>
<packaging>jar</packaging>
<name>Kyuubi Spark Kudu Connector</name>
<url>https://kyuubi.apache.org/</url>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalatestplus</groupId>
<artifactId>scalacheck-1-17_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.dimafeng</groupId>
<artifactId>testcontainers-scala-scalatest_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
<scope>test</scope>
</dependency>
<!--
Spark requires `commons-collections` and `commons-io` but got them from transitive
dependencies of `hadoop-client`. As we are using Hadoop Shaded Client, we need add
them explicitly. See more details at SPARK-33212.
-->
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<configuration>
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
<include>org.apache.kudu:kudu-client</include>
<include>org.apache.kyuubi:*</include>
<include>com.stumbleupon:async</include>
</includes>
</artifactSet>
<filters>
<filter>
<artifact>org.apache.kudu:kudu-client</artifact>
<excludes>
<exclude>META-INF/maven/**</exclude>
<exclude>META-INF/native/**</exclude>
<exclude>META-INF/native-image/**</exclude>
<exclude>MANIFEST.MF</exclude>
<exclude>LICENSE</exclude>
<exclude>LICENSE.txt</exclude>
<exclude>NOTICE</exclude>
<exclude>NOTICE.txt</exclude>
<exclude>*.properties</exclude>
<exclude>**/*.proto</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>org.apache.kudu</pattern>
<shadedPattern>${kyuubi.shade.packageName}.org.apache.kudu</shadedPattern>
<includes>
<include>org.apache.kudu.**</include>
</includes>
</relocation>
<relocation>
<pattern>com.stumbleupon:async</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.stumbleupon.async</shadedPattern>
<includes>
<include>com.stumbleupon.async.**</include>
</includes>
</relocation>
</relocations>
</configuration>
<executions>
<execution>
<goals>
<goal>shade</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<id>prepare-test-jar</id>
<goals>
<goal>test-jar</goal>
</goals>
<phase>test-compile</phase>
</execution>
</executions>
</plugin>
</plugins>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
</build>
</project>

64
extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/kudu-compose.yml
View File

@ -1,64 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
version: "3"
services:
kudu-master:
image: apache/kudu:1.15.0
hostname: kudu-master
ports:
- "7051"
- "8051"
command: ["master"]
environment:
- KUDU_MASTERS=kudu-master
kudu-tserver-1:
image: apache/kudu:1.15.0
depends_on:
- kudu-master
hostname: kudu-tserver-1
ports:
- "7050"
- "8050"
command: ["tserver"]
environment:
- KUDU_MASTERS=kudu-master
kudu-tserver-2:
image: apache/kudu:1.15.0
depends_on:
- kudu-master
hostname: kudu-tserver-2
ports:
- "7050"
- "8050"
command: [ "tserver" ]
environment:
- KUDU_MASTERS=kudu-master
kudu-tserver-3:
image: apache/kudu:1.15.0
depends_on:
- kudu-master
hostname: kudu-tserver-3
ports:
- "7050"
- "8050"
command: [ "tserver" ]
environment:
- KUDU_MASTERS=kudu-master

43
extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/log4j2-test.xml
View File

@ -1,43 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!-- Extra logging related to initialization of Log4j.
Set to debug or trace if log4j initialization is failing. -->
<Configuration status="WARN">
<Appenders>
<Console name="stdout" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} %p %c: %m%n"/>
<Filters>
<ThresholdFilter level="FATAL"/>
<RegexFilter regex=".*Thrift error occurred during processing of message.*" onMatch="DENY" onMismatch="NEUTRAL"/>
</Filters>
</Console>
<File name="file" fileName="target/unit-tests.log">
<PatternLayout pattern="%d{HH:mm:ss.SSS} %t %p %c{1}: %m%n"/>
<Filters>
<RegexFilter regex=".*Thrift error occurred during processing of message.*" onMatch="DENY" onMismatch="NEUTRAL"/>
</Filters>
</File>
</Appenders>
<Loggers>
<Root level="INFO">
<AppenderRef ref="stdout"/>
<AppenderRef ref="file"/>
</Root>
</Loggers>
</Configuration>

32
extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduClientSuite.scala
View File

@ -1,32 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.spark.connector.kudu
import org.apache.kudu.client.KuduClient
import org.apache.kyuubi.KyuubiFunSuite
class KuduClientSuite extends KyuubiFunSuite with KuduMixin {
test("kudu client") {
val builder = new KuduClient.KuduClientBuilder(kuduMasterUrl)
val kuduClient = builder.build()
assert(kuduClient.findLeaderMasterServer().getPort === kuduMasterPort)
}
}

41
extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala
View File

@ -1,41 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.spark.connector.kudu
import java.io.File
import com.dimafeng.testcontainers.{DockerComposeContainer, ExposedService, ForAllTestContainer}
import org.apache.kyuubi.{KyuubiFunSuite, Utils}
trait KuduMixin extends KyuubiFunSuite with ForAllTestContainer {
private val KUDU_MASTER_PORT = 7051
override val container: DockerComposeContainer =
DockerComposeContainer
.Def(
composeFiles =
new File(Utils.getContextOrKyuubiClassLoader.getResource("kudu-compose.yml").toURI),
exposedServices = ExposedService("kudu-master", KUDU_MASTER_PORT) :: Nil)
.createContainer()
def kuduMasterHost: String = container.getServiceHost("kudu-master", KUDU_MASTER_PORT)
def kuduMasterPort: Int = container.getServicePort("kudu-master", KUDU_MASTER_PORT)
def kuduMasterUrl: String = s"$kuduMasterHost:$kuduMasterPort"
}

8
pom.xml
View File

@ -169,7 +169,6 @@
<junit.version>4.13.2</junit.version>
<kafka.version>3.4.0</kafka.version>
<kubernetes-client.version>6.7.2</kubernetes-client.version>
<kudu.version>1.15.0</kudu.version>
<kyuubi-shaded-zookeeper.artifacts>kyuubi-shaded-zookeeper-34</kyuubi-shaded-zookeeper.artifacts>
<kyuubi-shaded-zookeeper.version>0.1.0</kyuubi-shaded-zookeeper.version>
<ldapsdk.version>6.0.5</ldapsdk.version>
@ -1461,11 +1460,6 @@
<version>${sqlite.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
<version>${kudu.version}</version>
</dependency>
<dependency>
<groupId>net.sf.py4j</groupId>
<artifactId>py4j</artifactId>
@ -2166,7 +2160,6 @@
<module>extensions/spark/kyuubi-extension-spark-common</module>
<module>extensions/spark/kyuubi-extension-spark-3-3</module>
<module>extensions/spark/kyuubi-spark-connector-hive</module>
<module>extensions/spark/kyuubi-spark-connector-kudu</module>
</modules>
<properties>
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow</maven.plugin.scalatest.exclude.tags>
@ -2178,7 +2171,6 @@
<modules>
<module>extensions/spark/kyuubi-extension-spark-3-4</module>
<module>extensions/spark/kyuubi-spark-connector-hive</module>
<module>extensions/spark/kyuubi-spark-connector-kudu</module>
</modules>
<properties>
<delta.version>2.4.0</delta.version>