diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 11a9580c4..6f575302e 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -67,7 +67,7 @@ jobs: MVN_OPT="-DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip" build/mvn clean install ${MVN_OPT} -Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.2,tpcds build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-1 -Pspark-3.1 - build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-kudu,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3 + build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3 build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-4 -Pspark-3.4 - name: Scalastyle with maven diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml index 3dfc7b986..09ced96a1 100644 --- a/dev/kyuubi-codecov/pom.xml +++ b/dev/kyuubi-codecov/pom.xml @@ -197,11 +197,6 @@ kyuubi-extension-spark-3-3_${scala.binary.version} ${project.version} - - org.apache.kyuubi - kyuubi-spark-connector-kudu_${scala.binary.version} - ${project.version} - org.apache.kyuubi kyuubi-spark-connector-hive_${scala.binary.version} diff --git a/extensions/spark/kyuubi-spark-connector-kudu/pom.xml b/extensions/spark/kyuubi-spark-connector-kudu/pom.xml deleted file mode 100644 index cb1cec449..000000000 --- a/extensions/spark/kyuubi-spark-connector-kudu/pom.xml +++ /dev/null @@ -1,225 +0,0 @@ - - - - 4.0.0 - - org.apache.kyuubi - kyuubi-parent - 1.8.0-SNAPSHOT - ../../../pom.xml - - - kyuubi-spark-connector-kudu_2.12 - jar - Kyuubi Spark Kudu Connector - https://kyuubi.apache.org/ - - - - org.scala-lang - scala-library - provided - - - - org.apache.logging.log4j - log4j-api - provided - - - - org.apache.logging.log4j - log4j-core - provided - - - - org.apache.spark - spark-sql_${scala.binary.version} - provided - - - - org.apache.hadoop - hadoop-client-api - provided - - - - org.apache.kudu - kudu-client - - - - org.apache.spark - spark-catalyst_${scala.binary.version} - test-jar - test - - - - org.scalatestplus - scalacheck-1-17_${scala.binary.version} - test - - - - com.dimafeng - testcontainers-scala-scalatest_${scala.binary.version} - test - - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - test-jar - test - - - - org.apache.kyuubi - kyuubi-common_${scala.binary.version} - ${project.version} - test - - - - org.apache.kyuubi - kyuubi-common_${scala.binary.version} - ${project.version} - test-jar - test - - - - org.apache.hadoop - hadoop-client-runtime - test - - - - - commons-collections - commons-collections - test - - - - commons-io - commons-io - test - - - - jakarta.xml.bind - jakarta.xml.bind-api - test - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - true - - - - - org.apache.maven.plugins - maven-shade-plugin - - false - - - org.apache.kudu:kudu-client - org.apache.kyuubi:* - com.stumbleupon:async - - - - - org.apache.kudu:kudu-client - - META-INF/maven/** - META-INF/native/** - META-INF/native-image/** - MANIFEST.MF - LICENSE - LICENSE.txt - NOTICE - NOTICE.txt - *.properties - **/*.proto - - - - - - org.apache.kudu - ${kyuubi.shade.packageName}.org.apache.kudu - - org.apache.kudu.** - - - - com.stumbleupon:async - ${kyuubi.shade.packageName}.com.stumbleupon.async - - com.stumbleupon.async.** - - - - - - - - shade - - package - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - prepare-test-jar - - test-jar - - test-compile - - - - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - - diff --git a/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/kudu-compose.yml b/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/kudu-compose.yml deleted file mode 100644 index 149cd5d47..000000000 --- a/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/kudu-compose.yml +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -version: "3" -services: - kudu-master: - image: apache/kudu:1.15.0 - hostname: kudu-master - ports: - - "7051" - - "8051" - command: ["master"] - environment: - - KUDU_MASTERS=kudu-master - - kudu-tserver-1: - image: apache/kudu:1.15.0 - depends_on: - - kudu-master - hostname: kudu-tserver-1 - ports: - - "7050" - - "8050" - command: ["tserver"] - environment: - - KUDU_MASTERS=kudu-master - - kudu-tserver-2: - image: apache/kudu:1.15.0 - depends_on: - - kudu-master - hostname: kudu-tserver-2 - ports: - - "7050" - - "8050" - command: [ "tserver" ] - environment: - - KUDU_MASTERS=kudu-master - - kudu-tserver-3: - image: apache/kudu:1.15.0 - depends_on: - - kudu-master - hostname: kudu-tserver-3 - ports: - - "7050" - - "8050" - command: [ "tserver" ] - environment: - - KUDU_MASTERS=kudu-master diff --git a/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/log4j2-test.xml b/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/log4j2-test.xml deleted file mode 100644 index bfc40dd6d..000000000 --- a/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/log4j2-test.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduClientSuite.scala b/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduClientSuite.scala deleted file mode 100644 index eebb4719c..000000000 --- a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduClientSuite.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.spark.connector.kudu - -import org.apache.kudu.client.KuduClient - -import org.apache.kyuubi.KyuubiFunSuite - -class KuduClientSuite extends KyuubiFunSuite with KuduMixin { - - test("kudu client") { - val builder = new KuduClient.KuduClientBuilder(kuduMasterUrl) - val kuduClient = builder.build() - - assert(kuduClient.findLeaderMasterServer().getPort === kuduMasterPort) - } -} diff --git a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala b/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala deleted file mode 100644 index dee09db38..000000000 --- a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.spark.connector.kudu - -import java.io.File - -import com.dimafeng.testcontainers.{DockerComposeContainer, ExposedService, ForAllTestContainer} - -import org.apache.kyuubi.{KyuubiFunSuite, Utils} - -trait KuduMixin extends KyuubiFunSuite with ForAllTestContainer { - - private val KUDU_MASTER_PORT = 7051 - - override val container: DockerComposeContainer = - DockerComposeContainer - .Def( - composeFiles = - new File(Utils.getContextOrKyuubiClassLoader.getResource("kudu-compose.yml").toURI), - exposedServices = ExposedService("kudu-master", KUDU_MASTER_PORT) :: Nil) - .createContainer() - - def kuduMasterHost: String = container.getServiceHost("kudu-master", KUDU_MASTER_PORT) - def kuduMasterPort: Int = container.getServicePort("kudu-master", KUDU_MASTER_PORT) - def kuduMasterUrl: String = s"$kuduMasterHost:$kuduMasterPort" -} diff --git a/pom.xml b/pom.xml index e3da6a6a8..9d30d0ec4 100644 --- a/pom.xml +++ b/pom.xml @@ -169,7 +169,6 @@ 4.13.2 3.4.0 6.7.2 - 1.15.0 kyuubi-shaded-zookeeper-34 0.1.0 6.0.5 @@ -1461,11 +1460,6 @@ ${sqlite.version} - - org.apache.kudu - kudu-client - ${kudu.version} - net.sf.py4j py4j @@ -2166,7 +2160,6 @@ extensions/spark/kyuubi-extension-spark-common extensions/spark/kyuubi-extension-spark-3-3 extensions/spark/kyuubi-spark-connector-hive - extensions/spark/kyuubi-spark-connector-kudu org.scalatest.tags.Slow @@ -2178,7 +2171,6 @@ extensions/spark/kyuubi-extension-spark-3-4 extensions/spark/kyuubi-spark-connector-hive - extensions/spark/kyuubi-spark-connector-kudu 2.4.0