apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #4870] Add kyuubi-util and kyuubi-util-scala modules

Browse Source 
### _Why are the changes needed?_

Close #4870

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #4872 from pan3793/util.

Closes #4870

0b9fe3cba [Cheng Pan] nit
ecc5ee4f2 [Cheng Pan] fix
63be7a20c [Cheng Pan] test
85363c187 [Cheng Pan] style
2227247dd [Cheng Pan] fix package
11d10a081 [Cheng Pan] Add kyuubi-util and kyuubi-util-scala modules

Authored-by: Cheng Pan <chengpan@apache.org>
Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Cheng Pan 2023-05-22 22:13:56 +08:00
parent d46e00f6fe
commit 01d80eb272
No known key found for this signature in database
GPG Key ID: 8001952629BCC75D
45 changed files with 305 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/spark/kyuubi-extension-spark-3-1/pom.xml
View File

@ -148,7 +148,7 @@
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
<include>org.apache.kyuubi:kyuubi-extension-spark-common_${scala.binary.version}</include>
<include>org.apache.kyuubi:*</include>
</includes>
</artifactSet>
</configuration>

2
extensions/spark/kyuubi-extension-spark-3-2/pom.xml
View File

@ -148,7 +148,7 @@
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
<include>org.apache.kyuubi:kyuubi-extension-spark-common_${scala.binary.version}</include>
<include>org.apache.kyuubi:*</include>
</includes>
</artifactSet>
</configuration>

2
extensions/spark/kyuubi-extension-spark-3-3/pom.xml
View File

@ -137,7 +137,7 @@
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
<include>org.apache.kyuubi:kyuubi-extension-spark-common_${scala.binary.version}</include>
<include>org.apache.kyuubi:*</include>
</includes>
</artifactSet>
</configuration>

5
extensions/spark/kyuubi-spark-authz/pom.xml
View File

@ -39,6 +39,11 @@
</properties>
<dependencies>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-util-scala_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.ranger</groupId>
<artifactId>ranger-plugins-common</artifactId>

1
extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala
View File

@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, View}
import org.apache.kyuubi.plugin.spark.authz.AccessControlException
import org.apache.kyuubi.plugin.spark.authz.util.ReservedKeys._
import org.apache.kyuubi.util.SemanticVersion
private[authz] object AuthZUtils {

74
extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/SemanticVersion.scala
View File

@ -1,74 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.plugin.spark.authz.util
/**
* Encapsulate a component Spark version for the convenience of version checks.
* Copy from org.apache.kyuubi.engine.ComponentVersion
*/
case class SemanticVersion(majorVersion: Int, minorVersion: Int) {
def isVersionAtMost(targetVersionString: String): Boolean = {
this.compareVersion(
targetVersionString,
(targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) =>
(runtimeMajor < targetMajor) || {
runtimeMajor == targetMajor && runtimeMinor <= targetMinor
})
}
def isVersionAtLeast(targetVersionString: String): Boolean = {
this.compareVersion(
targetVersionString,
(targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) =>
(runtimeMajor > targetMajor) || {
runtimeMajor == targetMajor && runtimeMinor >= targetMinor
})
}
def isVersionEqualTo(targetVersionString: String): Boolean = {
this.compareVersion(
targetVersionString,
(targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) =>
runtimeMajor == targetMajor && runtimeMinor == targetMinor)
}
def compareVersion(
targetVersionString: String,
callback: (Int, Int, Int, Int) => Boolean): Boolean = {
val targetVersion = SemanticVersion(targetVersionString)
val targetMajor = targetVersion.majorVersion
val targetMinor = targetVersion.minorVersion
callback(targetMajor, targetMinor, this.majorVersion, this.minorVersion)
}
override def toString: String = s"$majorVersion.$minorVersion"
}
object SemanticVersion {
def apply(versionString: String): SemanticVersion = {
"""^(\d+)\.(\d+)(\..*)?$""".r.findFirstMatchIn(versionString) match {
case Some(m) =>
SemanticVersion(m.group(1).toInt, m.group(2).toInt)
case None =>
throw new IllegalArgumentException(s"Tried to parse '$versionString' as a project" +
s" version string, but it could not find the major and minor version numbers.")
}
}
}

6
extensions/spark/kyuubi-spark-connector-common/pom.xml
View File

@ -31,6 +31,12 @@
<url>https://kyuubi.apache.org/</url>
<dependencies>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-util-scala_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>

74
extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SemanticVersion.scala
View File

@ -1,74 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.spark.connector.common
/**
* Encapsulate a component Spark version for the convenience of version checks.
* Copy from org.apache.kyuubi.engine.ComponentVersion
*/
case class SemanticVersion(majorVersion: Int, minorVersion: Int) {
def isVersionAtMost(targetVersionString: String): Boolean = {
this.compareVersion(
targetVersionString,
(targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) =>
(runtimeMajor < targetMajor) || {
runtimeMajor == targetMajor && runtimeMinor <= targetMinor
})
}
def isVersionAtLeast(targetVersionString: String): Boolean = {
this.compareVersion(
targetVersionString,
(targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) =>
(runtimeMajor > targetMajor) || {
runtimeMajor == targetMajor && runtimeMinor >= targetMinor
})
}
def isVersionEqualTo(targetVersionString: String): Boolean = {
this.compareVersion(
targetVersionString,
(targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) =>
runtimeMajor == targetMajor && runtimeMinor == targetMinor)
}
def compareVersion(
targetVersionString: String,
callback: (Int, Int, Int, Int) => Boolean): Boolean = {
val targetVersion = SemanticVersion(targetVersionString)
val targetMajor = targetVersion.majorVersion
val targetMinor = targetVersion.minorVersion
callback(targetMajor, targetMinor, this.majorVersion, this.minorVersion)
}
override def toString: String = s"$majorVersion.$minorVersion"
}
object SemanticVersion {
def apply(versionString: String): SemanticVersion = {
"""^(\d+)\.(\d+)(\..*)?$""".r.findFirstMatchIn(versionString) match {
case Some(m) =>
SemanticVersion(m.group(1).toInt, m.group(2).toInt)
case None =>
throw new IllegalArgumentException(s"Tried to parse '$versionString' as a project" +
s" version string, but it could not find the major and minor version numbers.")
}
}
}

2
extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SparkUtils.scala
View File

@ -19,6 +19,8 @@ package org.apache.kyuubi.spark.connector.common
import org.apache.spark.SPARK_VERSION
import org.apache.kyuubi.util.SemanticVersion
object SparkUtils {
def isSparkVersionAtMost(targetVersionString: String): Boolean = {

2
extensions/spark/kyuubi-spark-connector-hive/pom.xml
View File

@ -153,7 +153,7 @@
<artifactSet>
<includes>
<include>com.google.guava:guava</include>
<include>org.apache.kyuubi:kyuubi-spark-connector-common_${scala.binary.version}</include>
<include>org.apache.kyuubi:*</include>
</includes>
</artifactSet>
<relocations>

1
extensions/spark/kyuubi-spark-connector-kudu/pom.xml
View File

@ -157,6 +157,7 @@
<artifactSet>
<includes>
<include>org.apache.kudu:kudu-client</include>
<include>org.apache.kyuubi:*</include>
<include>com.stumbleupon:async</include>
</includes>
</artifactSet>

2
extensions/spark/kyuubi-spark-connector-tpcds/pom.xml
View File

@ -173,7 +173,7 @@
<includes>
<include>io.trino.tpcds:tpcds</include>
<include>com.google.guava:guava</include>
<include>org.apache.kyuubi:kyuubi-spark-connector-common_${scala.binary.version}</include>
<include>org.apache.kyuubi:*</include>
</includes>
</artifactSet>
<relocations>

2
extensions/spark/kyuubi-spark-connector-tpch/pom.xml
View File

@ -172,7 +172,7 @@
<includes>
<include>io.trino.tpch:tpch</include>
<include>com.google.guava:guava</include>
<include>org.apache.kyuubi:kyuubi-spark-connector-common_${scala.binary.version}</include>
<include>org.apache.kyuubi:*</include>
</includes>
</artifactSet>
<relocations>

5
extensions/spark/kyuubi-spark-lineage/pom.xml
View File

@ -31,6 +31,11 @@
<url>https://kyuubi.apache.org/</url>
<dependencies>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-util-scala_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>

74
extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SemanticVersion.scala
View File

@ -1,74 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.plugin.lineage.helper
/**
* Encapsulate a component (Kyuubi/Spark/Hive/Flink etc.) version
* for the convenience of version checks.
*/
case class SemanticVersion(majorVersion: Int, minorVersion: Int) {
def isVersionAtMost(targetVersionString: String): Boolean = {
this.compareVersion(
targetVersionString,
(targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) =>
(runtimeMajor < targetMajor) || {
runtimeMajor == targetMajor && runtimeMinor <= targetMinor
})
}
def isVersionAtLeast(targetVersionString: String): Boolean = {
this.compareVersion(
targetVersionString,
(targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) =>
(runtimeMajor > targetMajor) || {
runtimeMajor == targetMajor && runtimeMinor >= targetMinor
})
}
def isVersionEqualTo(targetVersionString: String): Boolean = {
this.compareVersion(
targetVersionString,
(targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) =>
runtimeMajor == targetMajor && runtimeMinor == targetMinor)
}
def compareVersion(
targetVersionString: String,
callback: (Int, Int, Int, Int) => Boolean): Boolean = {
val targetVersion = SemanticVersion(targetVersionString)
val targetMajor = targetVersion.majorVersion
val targetMinor = targetVersion.minorVersion
callback(targetMajor, targetMinor, this.majorVersion, this.minorVersion)
}
override def toString: String = s"$majorVersion.$minorVersion"
}
object SemanticVersion {
def apply(versionString: String): SemanticVersion = {
"""^(\d+)\.(\d+)(\..*)?$""".r.findFirstMatchIn(versionString) match {
case Some(m) =>
SemanticVersion(m.group(1).toInt, m.group(2).toInt)
case None =>
throw new IllegalArgumentException(s"Tried to parse '$versionString' as a project" +
s" version string, but it could not find the major and minor version numbers.")
}
}
}

2
extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkListenerHelper.scala
View File

@ -19,6 +19,8 @@ package org.apache.kyuubi.plugin.lineage.helper
import org.apache.spark.SPARK_VERSION
import org.apache.kyuubi.util.SemanticVersion
object SparkListenerHelper {
lazy val sparkMajorMinorVersion: (Int, Int) = {

2
externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatProvider.scala vendored
View File

@ -24,7 +24,7 @@ import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModul
import org.apache.kyuubi.{KyuubiException, Logging}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.reflection.DynConstructors
import org.apache.kyuubi.util.reflect.DynConstructors
trait ChatProvider {

4
externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala vendored
View File

@ -40,8 +40,8 @@ import org.apache.flink.table.gateway.service.session.Session
import org.apache.flink.util.JarUtils
import org.apache.kyuubi.{KyuubiException, Logging}
import org.apache.kyuubi.engine.SemanticVersion
import org.apache.kyuubi.reflection.{DynConstructors, DynFields, DynMethods}
import org.apache.kyuubi.util.SemanticVersion
import org.apache.kyuubi.util.reflect.{DynConstructors, DynFields, DynMethods}
object FlinkEngineUtils extends Logging {

2
externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkResultSet.scala vendored
View File

@ -24,7 +24,7 @@ import org.apache.flink.table.data.RowData
import org.apache.flink.table.gateway.api.results.ResultSet.ResultType
import org.apache.kyuubi.engine.flink.FlinkEngineUtils
import org.apache.kyuubi.reflection.DynMethods
import org.apache.kyuubi.util.reflect.DynMethods
class FlinkResultSet(resultSet: AnyRef) {

2
externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkSessionManager.scala vendored
View File

@ -22,7 +22,7 @@ import org.apache.flink.table.gateway.service.context.DefaultContext
import org.apache.flink.table.gateway.service.session.Session
import org.apache.kyuubi.engine.flink.FlinkEngineUtils
import org.apache.kyuubi.reflection.{DynConstructors, DynMethods}
import org.apache.kyuubi.util.reflect.{DynConstructors, DynMethods}
class FlinkSessionManager(engineContext: DefaultContext) {

2
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala vendored
View File

@ -26,7 +26,7 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.util.kvstore.KVIndex
import org.apache.kyuubi.Logging
import org.apache.kyuubi.engine.SemanticVersion
import org.apache.kyuubi.util.SemanticVersion
object KyuubiSparkUtil extends Logging {

2
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala vendored
View File

@ -31,10 +31,10 @@ import org.apache.kyuubi.{KyuubiSQLException, Logging}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiReservedKeys._
import org.apache.kyuubi.ha.client.{EngineServiceDiscovery, ServiceDiscovery}
import org.apache.kyuubi.reflection.DynConstructors
import org.apache.kyuubi.service.{Serverable, Service, TBinaryFrontendService}
import org.apache.kyuubi.service.TFrontendService._
import org.apache.kyuubi.util.KyuubiHadoopUtils
import org.apache.kyuubi.util.reflect.DynConstructors
class SparkTBinaryFrontendService(
override val serverable: Serverable)

2
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala vendored
View File

@ -38,7 +38,7 @@ import org.apache.spark.sql.types._
import org.apache.spark.sql.util.ArrowUtils
import org.apache.spark.util.Utils
import org.apache.kyuubi.reflection.DynMethods
import org.apache.kyuubi.util.reflect.DynMethods
object KyuubiArrowConverters extends SQLConfHelper with Logging {

2
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala vendored
View File

@ -35,7 +35,7 @@ import org.apache.spark.sql.types._
import org.apache.kyuubi.engine.spark.KyuubiSparkUtil
import org.apache.kyuubi.engine.spark.schema.RowSet
import org.apache.kyuubi.reflection.DynMethods
import org.apache.kyuubi.util.reflect.DynMethods
object SparkDatasetHelper extends Logging {

2
externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala vendored
View File

@ -40,7 +40,7 @@ import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.engine.spark.{SparkSQLEngine, WithSparkSQLEngine}
import org.apache.kyuubi.engine.spark.session.SparkSessionImpl
import org.apache.kyuubi.operation.SparkDataTypeTests
import org.apache.kyuubi.reflection.DynFields
import org.apache.kyuubi.util.reflect.DynFields
class SparkArrowbasedOperationSuite extends WithSparkSQLEngine with SparkDataTypeTests
with SparkMetricsTestUtils {

2
externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala vendored
View File

@ -32,13 +32,13 @@ import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.types._
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.engine.SemanticVersion
import org.apache.kyuubi.engine.spark.WithSparkSQLEngine
import org.apache.kyuubi.engine.spark.schema.SchemaHelper.TIMESTAMP_NTZ
import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim
import org.apache.kyuubi.operation.{HiveMetadataTests, SparkQueryTests}
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
import org.apache.kyuubi.util.KyuubiHadoopUtils
import org.apache.kyuubi.util.SemanticVersion
class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with SparkQueryTests {

19
kyuubi-common/pom.xml
View File

@ -32,9 +32,9 @@
<dependencies>
<dependency>
<groupId>com.vladsch.flexmark</groupId>
<artifactId>flexmark-all</artifactId>
<scope>test</scope>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-util-scala_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@ -128,6 +128,13 @@
<artifactId>HikariCP</artifactId>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-util-scala_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
@ -169,6 +176,12 @@
<artifactId>fliptables</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.vladsch.flexmark</groupId>
<artifactId>flexmark-all</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

2
kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala
View File

@ -28,7 +28,7 @@ import org.apache.logging.log4j.core.appender.{AbstractWriterAppender, ConsoleAp
import org.apache.logging.log4j.core.filter.AbstractFilter
import org.apache.logging.log4j.core.layout.PatternLayout
import org.apache.kyuubi.reflection.DynFields
import org.apache.kyuubi.util.reflect.DynFields
class Log4j2DivertAppender(
name: String,

2
kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLServer.scala
View File

@ -23,7 +23,7 @@ import javax.security.auth.callback.{Callback, CallbackHandler, NameCallback, Pa
import javax.security.sasl.{AuthorizeCallback, SaslException, SaslServer, SaslServerFactory}
import org.apache.kyuubi.KYUUBI_VERSION
import org.apache.kyuubi.engine.SemanticVersion
import org.apache.kyuubi.util.SemanticVersion
class PlainSASLServer(
handler: CallbackHandler,

2
kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLHelperSuite.scala
View File

@ -23,9 +23,9 @@ import org.apache.thrift.transport.{TSaslServerTransport, TSocket}
import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.engine.SemanticVersion
import org.apache.kyuubi.service.{NoopTBinaryFrontendServer, TBinaryFrontendService}
import org.apache.kyuubi.service.authentication.PlainSASLServer.SaslPlainProvider
import org.apache.kyuubi.util.SemanticVersion
class PlainSASLHelperSuite extends KyuubiFunSuite {

1
kyuubi-common/src/test/scala/org/apache/kyuubi/util/SparkVersionUtil.scala
View File

@ -17,7 +17,6 @@
package org.apache.kyuubi.util
import org.apache.kyuubi.engine.SemanticVersion
import org.apache.kyuubi.operation.HiveJDBCTestHelper
trait SparkVersionUtil {

2
kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala
View File

@ -31,10 +31,10 @@ import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ha.HighAvailabilityConf._
import org.apache.kyuubi.ha.client.{AuthTypes, RetryPolicies}
import org.apache.kyuubi.ha.client.RetryPolicies._
import org.apache.kyuubi.reflection.DynConstructors
import org.apache.kyuubi.shaded.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
import org.apache.kyuubi.shaded.curator.retry._
import org.apache.kyuubi.util.KyuubiHadoopUtils
import org.apache.kyuubi.util.reflect.DynConstructors
object ZookeeperClientProvider extends Logging {

2
kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala
View File

@ -26,10 +26,10 @@ import org.scalatest.time.SpanSugar._
import org.apache.kyuubi.{KYUUBI_VERSION, Utils, WithKyuubiServer, WithSimpleDFSService}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.KYUUBI_ENGINE_ENV_PREFIX
import org.apache.kyuubi.engine.SemanticVersion
import org.apache.kyuubi.jdbc.hive.KyuubiStatement
import org.apache.kyuubi.metrics.{MetricsConstants, MetricsSystem}
import org.apache.kyuubi.session.{KyuubiSessionImpl, SessionHandle}
import org.apache.kyuubi.util.SemanticVersion
import org.apache.kyuubi.zookeeper.ZookeeperConf
class KyuubiOperationPerUserSuite

63
kyuubi-util-scala/pom.xml Normal file
View File

@ -0,0 +1,63 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-parent</artifactId>
<version>1.8.0-SNAPSHOT</version>
</parent>
<artifactId>kyuubi-util-scala_2.12</artifactId>
<packaging>jar</packaging>
<name>Kyuubi Project Util Scala</name>
<url>https://kyuubi.apache.org/</url>
<dependencies>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-util</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<id>prepare-test-jar</id>
<goals>
<goal>test-jar</goal>
</goals>
<phase>test-compile</phase>
</execution>
</executions>
</plugin>
</plugins>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
</build>
</project>

5
kyuubi-common/src/main/scala/org/apache/kyuubi/engine/SemanticVersion.scala → kyuubi-util-scala/src/main/scala/org/apache/kyuubi/util/SemanticVersion.scala
View File

@ -15,11 +15,10 @@
* limitations under the License.
*/
package org.apache.kyuubi.engine
package org.apache.kyuubi.util
/**
* Encapsulate a component (Kyuubi/Spark/Hive/Flink etc.) version
* for the convenience of version checks.
* Encapsulate a component version for the convenience of version checks.
*/
case class SemanticVersion(majorVersion: Int, minorVersion: Int) {

29
kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/DeltaTest.java Normal file
View File

@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.tags;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.scalatest.TagAnnotation;
@TagAnnotation
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD, ElementType.TYPE})
public @interface DeltaTest {}

29
kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/IcebergTest.java Normal file
View File

@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.tags;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.scalatest.TagAnnotation;
@TagAnnotation
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD, ElementType.TYPE})
public @interface IcebergTest {}

29
kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PySparkTest.java Normal file
View File

@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.tags;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.scalatest.TagAnnotation;
@TagAnnotation
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD, ElementType.TYPE})
public @interface PySparkTest {}

12
kyuubi-common/src/test/scala/org/apache/kyuubi/engine/SemanticVersionSuite.scala → kyuubi-util-scala/src/test/java/org/apache/kyuubi/util/SemanticVersionSuite.scala
View File

@ -15,12 +15,14 @@
* limitations under the License.
*/
package org.apache.kyuubi.engine
import org.apache.kyuubi.KyuubiFunSuite
class SemanticVersionSuite extends KyuubiFunSuite {
package org.apache.kyuubi.util
// scalastyle:off
import org.scalatest.funsuite.AnyFunSuite
// scalastyle:on
// scalastyle:off
class SemanticVersionSuite extends AnyFunSuite {
// scalastyle:on
test("parse normal version") {
val version = SemanticVersion("1.12.4")
assert(version.majorVersion === 1)

78
kyuubi-util/pom.xml Normal file
View File

@ -0,0 +1,78 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-parent</artifactId>
<version>1.8.0-SNAPSHOT</version>
</parent>
<artifactId>kyuubi-util</artifactId>
<packaging>jar</packaging>
<name>Kyuubi Project Util</name>
<url>https://kyuubi.apache.org/</url>
<properties></properties>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skipTests>${skipTests}</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<executions>
<execution>
<id>attach-scaladocs</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
</build>
</project>

2
kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynClasses.java → kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynClasses.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.apache.kyuubi.reflection;
package org.apache.kyuubi.util.reflect;
import java.util.LinkedHashSet;
import java.util.Set;

2
kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynConstructors.java → kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynConstructors.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.apache.kyuubi.reflection;
package org.apache.kyuubi.util.reflect;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;

10
kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynFields.java → kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynFields.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.apache.kyuubi.reflection;
package org.apache.kyuubi.util.reflect;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
@ -25,7 +25,6 @@ import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.lang3.builder.ToStringBuilder;
/** Copied from iceberg-common */
public class DynFields {
@ -66,11 +65,8 @@ public class DynFields {
@Override
public String toString() {
return new ToStringBuilder(this)
.append("class", field.getDeclaringClass().toString())
.append("name", name)
.append("type", field.getType())
.toString();
return String.format(
"DynFields{class=%s,name=%s,type=%s}", field.getDeclaringClass(), name, field.getType());
}
/**

2
kyuubi-common/src/main/java/org/apache/kyuubi/reflection/DynMethods.java → kyuubi-util/src/main/java/org/apache/kyuubi/util/reflect/DynMethods.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.apache.kyuubi.reflection;
package org.apache.kyuubi.util.reflect;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

2
pom.xml
View File

@ -79,6 +79,8 @@
<module>kyuubi-metrics</module>
<module>kyuubi-rest-client</module>
<module>kyuubi-server</module>
<module>kyuubi-util</module>
<module>kyuubi-util-scala</module>
<module>kyuubi-zookeeper</module>
</modules>