diff --git a/extensions/spark/kyuubi-extension-spark-3-1/pom.xml b/extensions/spark/kyuubi-extension-spark-3-1/pom.xml index a36dffaef..eb56e435b 100644 --- a/extensions/spark/kyuubi-extension-spark-3-1/pom.xml +++ b/extensions/spark/kyuubi-extension-spark-3-1/pom.xml @@ -148,7 +148,7 @@ false - org.apache.kyuubi:kyuubi-extension-spark-common_${scala.binary.version} + org.apache.kyuubi:* diff --git a/extensions/spark/kyuubi-extension-spark-3-2/pom.xml b/extensions/spark/kyuubi-extension-spark-3-2/pom.xml index 3f8019fa9..c8caed835 100644 --- a/extensions/spark/kyuubi-extension-spark-3-2/pom.xml +++ b/extensions/spark/kyuubi-extension-spark-3-2/pom.xml @@ -148,7 +148,7 @@ false - org.apache.kyuubi:kyuubi-extension-spark-common_${scala.binary.version} + org.apache.kyuubi:* diff --git a/extensions/spark/kyuubi-extension-spark-3-3/pom.xml b/extensions/spark/kyuubi-extension-spark-3-3/pom.xml index ca729a781..98c1cca02 100644 --- a/extensions/spark/kyuubi-extension-spark-3-3/pom.xml +++ b/extensions/spark/kyuubi-extension-spark-3-3/pom.xml @@ -137,7 +137,7 @@ false - org.apache.kyuubi:kyuubi-extension-spark-common_${scala.binary.version} + org.apache.kyuubi:* diff --git a/extensions/spark/kyuubi-spark-authz/pom.xml b/extensions/spark/kyuubi-spark-authz/pom.xml index 27417109d..7f95a6b05 100644 --- a/extensions/spark/kyuubi-spark-authz/pom.xml +++ b/extensions/spark/kyuubi-spark-authz/pom.xml @@ -39,6 +39,11 @@ + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + org.apache.ranger ranger-plugins-common diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala index 5773e1c93..46bc1fa3c 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, View} import org.apache.kyuubi.plugin.spark.authz.AccessControlException import org.apache.kyuubi.plugin.spark.authz.util.ReservedKeys._ +import org.apache.kyuubi.util.SemanticVersion private[authz] object AuthZUtils { diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/SemanticVersion.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/SemanticVersion.scala deleted file mode 100644 index 4d7e89725..000000000 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/SemanticVersion.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.plugin.spark.authz.util - -/** - * Encapsulate a component Spark version for the convenience of version checks. - * Copy from org.apache.kyuubi.engine.ComponentVersion - */ -case class SemanticVersion(majorVersion: Int, minorVersion: Int) { - - def isVersionAtMost(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor < targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor <= targetMinor - }) - } - - def isVersionAtLeast(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor > targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor >= targetMinor - }) - } - - def isVersionEqualTo(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - runtimeMajor == targetMajor && runtimeMinor == targetMinor) - } - - def compareVersion( - targetVersionString: String, - callback: (Int, Int, Int, Int) => Boolean): Boolean = { - val targetVersion = SemanticVersion(targetVersionString) - val targetMajor = targetVersion.majorVersion - val targetMinor = targetVersion.minorVersion - callback(targetMajor, targetMinor, this.majorVersion, this.minorVersion) - } - - override def toString: String = s"$majorVersion.$minorVersion" -} - -object SemanticVersion { - - def apply(versionString: String): SemanticVersion = { - """^(\d+)\.(\d+)(\..*)?$""".r.findFirstMatchIn(versionString) match { - case Some(m) => - SemanticVersion(m.group(1).toInt, m.group(2).toInt) - case None => - throw new IllegalArgumentException(s"Tried to parse '$versionString' as a project" + - s" version string, but it could not find the major and minor version numbers.") - } - } -} diff --git a/extensions/spark/kyuubi-spark-connector-common/pom.xml b/extensions/spark/kyuubi-spark-connector-common/pom.xml index e36361753..132f8ea83 100644 --- a/extensions/spark/kyuubi-spark-connector-common/pom.xml +++ b/extensions/spark/kyuubi-spark-connector-common/pom.xml @@ -31,6 +31,12 @@ https://kyuubi.apache.org/ + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + + org.scala-lang scala-library diff --git a/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SemanticVersion.scala b/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SemanticVersion.scala deleted file mode 100644 index 200937ca6..000000000 --- a/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SemanticVersion.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.spark.connector.common - -/** - * Encapsulate a component Spark version for the convenience of version checks. - * Copy from org.apache.kyuubi.engine.ComponentVersion - */ -case class SemanticVersion(majorVersion: Int, minorVersion: Int) { - - def isVersionAtMost(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor < targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor <= targetMinor - }) - } - - def isVersionAtLeast(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor > targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor >= targetMinor - }) - } - - def isVersionEqualTo(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - runtimeMajor == targetMajor && runtimeMinor == targetMinor) - } - - def compareVersion( - targetVersionString: String, - callback: (Int, Int, Int, Int) => Boolean): Boolean = { - val targetVersion = SemanticVersion(targetVersionString) - val targetMajor = targetVersion.majorVersion - val targetMinor = targetVersion.minorVersion - callback(targetMajor, targetMinor, this.majorVersion, this.minorVersion) - } - - override def toString: String = s"$majorVersion.$minorVersion" -} - -object SemanticVersion { - - def apply(versionString: String): SemanticVersion = { - """^(\d+)\.(\d+)(\..*)?$""".r.findFirstMatchIn(versionString) match { - case Some(m) => - SemanticVersion(m.group(1).toInt, m.group(2).toInt) - case None => - throw new IllegalArgumentException(s"Tried to parse '$versionString' as a project" + - s" version string, but it could not find the major and minor version numbers.") - } - } -} diff --git a/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SparkUtils.scala b/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SparkUtils.scala index c1a659fbf..a6c1624c1 100644 --- a/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SparkUtils.scala +++ b/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SparkUtils.scala @@ -19,6 +19,8 @@ package org.apache.kyuubi.spark.connector.common import org.apache.spark.SPARK_VERSION +import org.apache.kyuubi.util.SemanticVersion + object SparkUtils { def isSparkVersionAtMost(targetVersionString: String): Boolean = { diff --git a/extensions/spark/kyuubi-spark-connector-hive/pom.xml b/extensions/spark/kyuubi-spark-connector-hive/pom.xml index b75db929d..8a182748a 100644 --- a/extensions/spark/kyuubi-spark-connector-hive/pom.xml +++ b/extensions/spark/kyuubi-spark-connector-hive/pom.xml @@ -153,7 +153,7 @@ com.google.guava:guava - org.apache.kyuubi:kyuubi-spark-connector-common_${scala.binary.version} + org.apache.kyuubi:* diff --git a/extensions/spark/kyuubi-spark-connector-kudu/pom.xml b/extensions/spark/kyuubi-spark-connector-kudu/pom.xml index 97356cd93..cb1cec449 100644 --- a/extensions/spark/kyuubi-spark-connector-kudu/pom.xml +++ b/extensions/spark/kyuubi-spark-connector-kudu/pom.xml @@ -157,6 +157,7 @@ org.apache.kudu:kudu-client + org.apache.kyuubi:* com.stumbleupon:async diff --git a/extensions/spark/kyuubi-spark-connector-tpcds/pom.xml b/extensions/spark/kyuubi-spark-connector-tpcds/pom.xml index e9b867739..4bad96dda 100644 --- a/extensions/spark/kyuubi-spark-connector-tpcds/pom.xml +++ b/extensions/spark/kyuubi-spark-connector-tpcds/pom.xml @@ -173,7 +173,7 @@ io.trino.tpcds:tpcds com.google.guava:guava - org.apache.kyuubi:kyuubi-spark-connector-common_${scala.binary.version} + org.apache.kyuubi:* diff --git a/extensions/spark/kyuubi-spark-connector-tpch/pom.xml b/extensions/spark/kyuubi-spark-connector-tpch/pom.xml index 5b418e200..fe5721c0f 100644 --- a/extensions/spark/kyuubi-spark-connector-tpch/pom.xml +++ b/extensions/spark/kyuubi-spark-connector-tpch/pom.xml @@ -172,7 +172,7 @@ io.trino.tpch:tpch com.google.guava:guava - org.apache.kyuubi:kyuubi-spark-connector-common_${scala.binary.version} + org.apache.kyuubi:* diff --git a/extensions/spark/kyuubi-spark-lineage/pom.xml b/extensions/spark/kyuubi-spark-lineage/pom.xml index 8583dfec0..2a7ba7773 100644 --- a/extensions/spark/kyuubi-spark-lineage/pom.xml +++ b/extensions/spark/kyuubi-spark-lineage/pom.xml @@ -31,6 +31,11 @@ https://kyuubi.apache.org/ + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + org.apache.spark spark-sql_${scala.binary.version} diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SemanticVersion.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SemanticVersion.scala deleted file mode 100644 index a4a8b2e0e..000000000 --- a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SemanticVersion.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.plugin.lineage.helper - -/** - * Encapsulate a component (Kyuubi/Spark/Hive/Flink etc.) version - * for the convenience of version checks. - */ -case class SemanticVersion(majorVersion: Int, minorVersion: Int) { - - def isVersionAtMost(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor < targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor <= targetMinor - }) - } - - def isVersionAtLeast(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor > targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor >= targetMinor - }) - } - - def isVersionEqualTo(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - runtimeMajor == targetMajor && runtimeMinor == targetMinor) - } - - def compareVersion( - targetVersionString: String, - callback: (Int, Int, Int, Int) => Boolean): Boolean = { - val targetVersion = SemanticVersion(targetVersionString) - val targetMajor = targetVersion.majorVersion - val targetMinor = targetVersion.minorVersion - callback(targetMajor, targetMinor, this.majorVersion, this.minorVersion) - } - - override def toString: String = s"$majorVersion.$minorVersion" -} - -object SemanticVersion { - - def apply(versionString: String): SemanticVersion = { - """^(\d+)\.(\d+)(\..*)?$""".r.findFirstMatchIn(versionString) match { - case Some(m) => - SemanticVersion(m.group(1).toInt, m.group(2).toInt) - case None => - throw new IllegalArgumentException(s"Tried to parse '$versionString' as a project" + - s" version string, but it could not find the major and minor version numbers.") - } - } -} diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkListenerHelper.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkListenerHelper.scala index f2808a4e9..57db55a1e 100644 --- a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkListenerHelper.scala +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkListenerHelper.scala @@ -19,6 +19,8 @@ package org.apache.kyuubi.plugin.lineage.helper import org.apache.spark.SPARK_VERSION +import org.apache.kyuubi.util.SemanticVersion + object SparkListenerHelper { lazy val sparkMajorMinorVersion: (Int, Int) = { diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatProvider.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatProvider.scala index f9fa8bb5a..06d719380 100644 --- a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatProvider.scala +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatProvider.scala @@ -24,7 +24,7 @@ import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModul import org.apache.kyuubi.{KyuubiException, Logging} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.reflection.DynConstructors +import org.apache.kyuubi.util.reflect.DynConstructors trait ChatProvider { diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala index f9289ea81..66d9ed2c5 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala @@ -40,8 +40,8 @@ import org.apache.flink.table.gateway.service.session.Session import org.apache.flink.util.JarUtils import org.apache.kyuubi.{KyuubiException, Logging} -import org.apache.kyuubi.engine.SemanticVersion -import org.apache.kyuubi.reflection.{DynConstructors, DynFields, DynMethods} +import org.apache.kyuubi.util.SemanticVersion +import org.apache.kyuubi.util.reflect.{DynConstructors, DynFields, DynMethods} object FlinkEngineUtils extends Logging { diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkResultSet.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkResultSet.scala index e3dd0f081..a9c76082d 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkResultSet.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkResultSet.scala @@ -24,7 +24,7 @@ import org.apache.flink.table.data.RowData import org.apache.flink.table.gateway.api.results.ResultSet.ResultType import org.apache.kyuubi.engine.flink.FlinkEngineUtils -import org.apache.kyuubi.reflection.DynMethods +import org.apache.kyuubi.util.reflect.DynMethods class FlinkResultSet(resultSet: AnyRef) { diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkSessionManager.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkSessionManager.scala index e34819dd6..ce5bdfbd9 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkSessionManager.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkSessionManager.scala @@ -22,7 +22,7 @@ import org.apache.flink.table.gateway.service.context.DefaultContext import org.apache.flink.table.gateway.service.session.Session import org.apache.kyuubi.engine.flink.FlinkEngineUtils -import org.apache.kyuubi.reflection.{DynConstructors, DynMethods} +import org.apache.kyuubi.util.reflect.{DynConstructors, DynMethods} class FlinkSessionManager(engineContext: DefaultContext) { diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala index 2c3e7195c..56bb2d69f 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.util.kvstore.KVIndex import org.apache.kyuubi.Logging -import org.apache.kyuubi.engine.SemanticVersion +import org.apache.kyuubi.util.SemanticVersion object KyuubiSparkUtil extends Logging { diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala index 781ac2d07..c2563b32b 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala @@ -31,10 +31,10 @@ import org.apache.kyuubi.{KyuubiSQLException, Logging} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiReservedKeys._ import org.apache.kyuubi.ha.client.{EngineServiceDiscovery, ServiceDiscovery} -import org.apache.kyuubi.reflection.DynConstructors import org.apache.kyuubi.service.{Serverable, Service, TBinaryFrontendService} import org.apache.kyuubi.service.TFrontendService._ import org.apache.kyuubi.util.KyuubiHadoopUtils +import org.apache.kyuubi.util.reflect.DynConstructors class SparkTBinaryFrontendService( override val serverable: Serverable) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala index 5930dcdfc..24b0ac22e 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala @@ -38,7 +38,7 @@ import org.apache.spark.sql.types._ import org.apache.spark.sql.util.ArrowUtils import org.apache.spark.util.Utils -import org.apache.kyuubi.reflection.DynMethods +import org.apache.kyuubi.util.reflect.DynMethods object KyuubiArrowConverters extends SQLConfHelper with Logging { diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala index 8f8aef560..06eb9a144 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.types._ import org.apache.kyuubi.engine.spark.KyuubiSparkUtil import org.apache.kyuubi.engine.spark.schema.RowSet -import org.apache.kyuubi.reflection.DynMethods +import org.apache.kyuubi.util.reflect.DynMethods object SparkDatasetHelper extends Logging { diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala index 9273ab879..fc53cc41a 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala @@ -40,7 +40,7 @@ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.engine.spark.{SparkSQLEngine, WithSparkSQLEngine} import org.apache.kyuubi.engine.spark.session.SparkSessionImpl import org.apache.kyuubi.operation.SparkDataTypeTests -import org.apache.kyuubi.reflection.DynFields +import org.apache.kyuubi.util.reflect.DynFields class SparkArrowbasedOperationSuite extends WithSparkSQLEngine with SparkDataTypeTests with SparkMetricsTestUtils { diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala index af514ceb3..cf5044056 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala @@ -32,13 +32,13 @@ import org.apache.spark.sql.catalyst.analysis.FunctionRegistry import org.apache.spark.sql.types._ import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.engine.SemanticVersion import org.apache.kyuubi.engine.spark.WithSparkSQLEngine import org.apache.kyuubi.engine.spark.schema.SchemaHelper.TIMESTAMP_NTZ import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim import org.apache.kyuubi.operation.{HiveMetadataTests, SparkQueryTests} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.util.KyuubiHadoopUtils +import org.apache.kyuubi.util.SemanticVersion class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with SparkQueryTests { diff --git a/kyuubi-common/pom.xml b/kyuubi-common/pom.xml index d62761d72..1ff8bea98 100644 --- a/kyuubi-common/pom.xml +++ b/kyuubi-common/pom.xml @@ -32,9 +32,9 @@ - com.vladsch.flexmark - flexmark-all - test + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} @@ -128,6 +128,13 @@ HikariCP + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + test-jar + + org.apache.hadoop hadoop-minikdc @@ -169,6 +176,12 @@ fliptables test + + + com.vladsch.flexmark + flexmark-all + test + diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala index 1c6c1dcc6..1e5684d4c 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala @@ -28,7 +28,7 @@ import org.apache.logging.log4j.core.appender.{AbstractWriterAppender, ConsoleAp import org.apache.logging.log4j.core.filter.AbstractFilter import org.apache.logging.log4j.core.layout.PatternLayout -import org.apache.kyuubi.reflection.DynFields +import org.apache.kyuubi.util.reflect.DynFields class Log4j2DivertAppender( name: String, diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLServer.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLServer.scala index 8e84c9f81..63cc4c0f3 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLServer.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLServer.scala @@ -23,7 +23,7 @@ import javax.security.auth.callback.{Callback, CallbackHandler, NameCallback, Pa import javax.security.sasl.{AuthorizeCallback, SaslException, SaslServer, SaslServerFactory} import org.apache.kyuubi.KYUUBI_VERSION -import org.apache.kyuubi.engine.SemanticVersion +import org.apache.kyuubi.util.SemanticVersion class PlainSASLServer( handler: CallbackHandler, diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLHelperSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLHelperSuite.scala index 94a61f693..795eaa0d6 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLHelperSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLHelperSuite.scala @@ -23,9 +23,9 @@ import org.apache.thrift.transport.{TSaslServerTransport, TSocket} import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.engine.SemanticVersion import org.apache.kyuubi.service.{NoopTBinaryFrontendServer, TBinaryFrontendService} import org.apache.kyuubi.service.authentication.PlainSASLServer.SaslPlainProvider +import org.apache.kyuubi.util.SemanticVersion class PlainSASLHelperSuite extends KyuubiFunSuite { diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/util/SparkVersionUtil.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/util/SparkVersionUtil.scala index 785015cc3..098de23af 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/util/SparkVersionUtil.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/util/SparkVersionUtil.scala @@ -17,7 +17,6 @@ package org.apache.kyuubi.util -import org.apache.kyuubi.engine.SemanticVersion import org.apache.kyuubi.operation.HiveJDBCTestHelper trait SparkVersionUtil { diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala index d9161f3cd..eaffedd12 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala @@ -31,10 +31,10 @@ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.ha.HighAvailabilityConf._ import org.apache.kyuubi.ha.client.{AuthTypes, RetryPolicies} import org.apache.kyuubi.ha.client.RetryPolicies._ -import org.apache.kyuubi.reflection.DynConstructors import org.apache.kyuubi.shaded.curator.framework.{CuratorFramework, CuratorFrameworkFactory} import org.apache.kyuubi.shaded.curator.retry._ import org.apache.kyuubi.util.KyuubiHadoopUtils +import org.apache.kyuubi.util.reflect.DynConstructors object ZookeeperClientProvider extends Logging { diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala index 2ae2340b2..fb6f4efa7 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala @@ -26,10 +26,10 @@ import org.scalatest.time.SpanSugar._ import org.apache.kyuubi.{KYUUBI_VERSION, Utils, WithKyuubiServer, WithSimpleDFSService} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.KYUUBI_ENGINE_ENV_PREFIX -import org.apache.kyuubi.engine.SemanticVersion import org.apache.kyuubi.jdbc.hive.KyuubiStatement import org.apache.kyuubi.metrics.{MetricsConstants, MetricsSystem} import org.apache.kyuubi.session.{KyuubiSessionImpl, SessionHandle} +import org.apache.kyuubi.util.SemanticVersion import org.apache.kyuubi.zookeeper.ZookeeperConf class KyuubiOperationPerUserSuite diff --git a/kyuubi-util-scala/pom.xml b/kyuubi-util-scala/pom.xml new file mode 100644 index 000000000..19df6d78e --- /dev/null +++ b/kyuubi-util-scala/pom.xml @@ -0,0 +1,63 @@ + + + + 4.0.0 + + org.apache.kyuubi + kyuubi-parent + 1.8.0-SNAPSHOT + + + kyuubi-util-scala_2.12 + jar + Kyuubi Project Util Scala + https://kyuubi.apache.org/ + + + + org.apache.kyuubi + kyuubi-util + ${project.version} + + + org.scala-lang + scala-library + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + prepare-test-jar + + test-jar + + test-compile + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/SemanticVersion.scala b/kyuubi-util-scala/src/main/scala/org/apache/kyuubi/util/SemanticVersion.scala similarity index 95% rename from kyuubi-common/src/main/scala/org/apache/kyuubi/engine/SemanticVersion.scala rename to kyuubi-util-scala/src/main/scala/org/apache/kyuubi/util/SemanticVersion.scala index a36896ac9..2b4e718c9 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/SemanticVersion.scala +++ b/kyuubi-util-scala/src/main/scala/org/apache/kyuubi/util/SemanticVersion.scala @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.kyuubi.engine +package org.apache.kyuubi.util /** - * Encapsulate a component (Kyuubi/Spark/Hive/Flink etc.) version - * for the convenience of version checks. + * Encapsulate a component version for the convenience of version checks. */ case class SemanticVersion(majorVersion: Int, minorVersion: Int) { diff --git a/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/DeltaTest.java b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/DeltaTest.java new file mode 100644 index 000000000..1b4b6d00c --- /dev/null +++ b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/DeltaTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.scalatest.TagAnnotation; + +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface DeltaTest {} diff --git a/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/IcebergTest.java b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/IcebergTest.java new file mode 100644 index 000000000..bb598ad68 --- /dev/null +++ b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/IcebergTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.scalatest.TagAnnotation; + +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface IcebergTest {} diff --git a/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PySparkTest.java b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PySparkTest.java new file mode 100644 index 000000000..1b7aa97b4 --- /dev/null +++ b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PySparkTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.scalatest.TagAnnotation; + +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface PySparkTest {} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/engine/SemanticVersionSuite.scala b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/util/SemanticVersionSuite.scala similarity index 93% rename from kyuubi-common/src/test/scala/org/apache/kyuubi/engine/SemanticVersionSuite.scala rename to kyuubi-util-scala/src/test/java/org/apache/kyuubi/util/SemanticVersionSuite.scala index 28427bdc8..d7eef2fc4 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/engine/SemanticVersionSuite.scala +++ b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/util/SemanticVersionSuite.scala @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.kyuubi.engine - -import org.apache.kyuubi.KyuubiFunSuite - -class SemanticVersionSuite extends KyuubiFunSuite { +package org.apache.kyuubi.util +// scalastyle:off +import org.scalatest.funsuite.AnyFunSuite +// scalastyle:on +// scalastyle:off +class SemanticVersionSuite extends AnyFunSuite { +// scalastyle:on test("parse normal version") { val version = SemanticVersion("1.12.4") assert(version.majorVersion === 1) diff --git a/kyuubi-util/pom.xml b/kyuubi-util/pom.xml new file mode 100644 index 000000000..dca500191 --- /dev/null +++ b/kyuubi-util/pom.xml @@ -0,0 +1,78 @@ + + + + 4.0.0 + + org.apache.kyuubi + kyuubi-parent + 1.8.0-SNAPSHOT + + + kyuubi-util + jar + Kyuubi Project Util + https://kyuubi.apache.org/ + + + + + + org.slf4j + slf4j-api + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${skipTests} + + + + net.alchim31.maven + scala-maven-plugin + + + attach-scaladocs + none + + + + + org.scalatest + scalatest-maven-plugin + + true + + + + org.scalastyle + scalastyle-maven-plugin + + true + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + diff --git a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynClasses.java b/kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynClasses.java similarity index 98% rename from kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynClasses.java rename to kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynClasses.java index 05661e6a6..78bdd54af 100644 --- a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynClasses.java +++ b/kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynClasses.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.kyuubi.reflection; +package org.apache.kyuubi.util.reflect; import java.util.LinkedHashSet; import java.util.Set; diff --git a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynConstructors.java b/kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynConstructors.java similarity index 99% rename from kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynConstructors.java rename to kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynConstructors.java index 59c79b885..daf55363a 100644 --- a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynConstructors.java +++ b/kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynConstructors.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.kyuubi.reflection; +package org.apache.kyuubi.util.reflect; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; diff --git a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynFields.java b/kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynFields.java similarity index 97% rename from kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynFields.java rename to kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynFields.java index 9430d54e9..4c7af9036 100644 --- a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynFields.java +++ b/kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynFields.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.kyuubi.reflection; +package org.apache.kyuubi.util.reflect; import java.lang.reflect.Field; import java.lang.reflect.Modifier; @@ -25,7 +25,6 @@ import java.security.AccessController; import java.security.PrivilegedAction; import java.util.HashSet; import java.util.Set; -import org.apache.commons.lang3.builder.ToStringBuilder; /** Copied from iceberg-common */ public class DynFields { @@ -66,11 +65,8 @@ public class DynFields { @Override public String toString() { - return new ToStringBuilder(this) - .append("class", field.getDeclaringClass().toString()) - .append("name", name) - .append("type", field.getType()) - .toString(); + return String.format( + "DynFields{class=%s,name=%s,type=%s}", field.getDeclaringClass(), name, field.getType()); } /** diff --git a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynMethods.java b/kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynMethods.java similarity index 99% rename from kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynMethods.java rename to kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynMethods.java index 85264191c..ca059df0b 100644 --- a/kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynMethods.java +++ b/kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynMethods.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.kyuubi.reflection; +package org.apache.kyuubi.util.reflect; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; diff --git a/pom.xml b/pom.xml index ae5c46140..fad1d1bdc 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,8 @@ kyuubi-metrics kyuubi-rest-client kyuubi-server + kyuubi-util + kyuubi-util-scala kyuubi-zookeeper