apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Revert "[KYUUBI #488] Add kyuubi-shaded module to avoid conflicts with external jars"

Browse Source 
This reverts commit e058b87dd2.
This commit is contained in:
Cheng Pan 2021-04-10 00:16:04 +08:00
parent e058b87dd2
commit b7ac3cb48c
81 changed files with 292 additions and 457 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/master.yml vendored
View File

@ -67,8 +67,6 @@ jobs:
key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-io-
- name: install kyuubi-shaded jars
run: mvn clean install -pl :kyuubi-shaded-curator,:kyuubi-shaded-hive-service-rpc -am -Pkyuubi-shaded
- name: Build with Maven
run: |
mvn clean install ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -V

1
.travis.yml
View File

@ -47,7 +47,6 @@ matrix:
install:
- mvn --version
- mvn clean install -pl :kyuubi-shaded-curator,:kyuubi-shaded-hive-service-rpc -am -Pkyuubi-shaded
script:
- mvn clean install $PROFILE -Dmaven.plugin.scalatest.exclude.tags=$EXCLUDE_TAGS -Dmaven.javadoc.skip=true -V

6
docs/tools/building.md
View File

@ -8,15 +8,9 @@
## Building Kyuubi with Apache Maven
Apache Curator and Hive Service RPC are shaded to avoid conflicts with external jars.
You need install these shaded modules locally at first.
**Kyuubi** is built based on [Apache Maven](http://maven.apache.org),
```bash
./build/mvn clean install -pl :kyuubi-shaded-curator,:kyuubi-shaded-hive-service-rpc -am -Pkyuubi-shaded
./build/mvn clean package -DskipTests
```

14
externals/kyuubi-spark-sql-engine/pom.xml vendored
View File

@ -98,12 +98,6 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-service-rpc</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-llap-client</artifactId>
@ -134,6 +128,12 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
@ -167,8 +167,6 @@
<includes>
<include>org.apache.kyuubi:kyuubi-common</include>
<include>org.apache.kyuubi:kyuubi-ha</include>
<include>org.apache.kyuubi:kyuubi-shaded-curator</include>
<include>org.apache.kyuubi:kyuubi-shaded-hive-service-rpc</include>
</includes>
</artifactSet>
</configuration>

2
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala vendored
View File

@ -21,6 +21,7 @@ import java.time.ZoneId
import java.util.regex.Pattern
import org.apache.commons.lang3.StringUtils
import org.apache.hive.service.rpc.thrift.{TRowSet, TTableSchema}
import org.apache.spark.sql.{Row, SparkSession}
import org.apache.spark.sql.types.StructType
@ -34,7 +35,6 @@ import org.apache.kyuubi.operation.OperationType.OperationType
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.schema.{RowSet, SchemaHelper}
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TRowSet, TTableSchema}
abstract class SparkOperation(spark: SparkSession, opType: OperationType, session: Session)
extends AbstractOperation(opType, session) {

2
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala vendored
View File

@ -17,6 +17,7 @@
package org.apache.kyuubi.engine.spark.session
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.kyuubi.KyuubiSQLException
@ -25,7 +26,6 @@ import org.apache.kyuubi.engine.ShareLevel
import org.apache.kyuubi.engine.spark.SparkSQLEngine
import org.apache.kyuubi.engine.spark.operation.SparkSQLOperationManager
import org.apache.kyuubi.session._
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
/**
* A [[SessionManager]] constructed with [[SparkSession]] which give it the ability to talk with

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala vendored
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.engine.spark.session
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.kyuubi.session.{AbstractSession, SessionManager}
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
class SparkSessionImpl(
protocol: TProtocolVersion,

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/schema/RowSet.scala vendored
View File

@ -30,11 +30,10 @@ import java.util.{Date, Locale}
import scala.collection.JavaConverters._
import scala.language.implicitConversions
import org.apache.hive.service.rpc.thrift._
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
import org.apache.kyuubi.shade.hive.service.rpc.thrift._
object RowSet {
def toTRowSet(

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/schema/SchemaHelper.scala vendored
View File

@ -21,10 +21,9 @@ import java.util.Collections
import scala.collection.JavaConverters._
import org.apache.hive.service.rpc.thrift._
import org.apache.spark.sql.types._
import org.apache.kyuubi.shade.hive.service.rpc.thrift._
object SchemaHelper {
def toTTypeId(typ: DataType): TTypeId = typ match {

3
externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/WithDiscoverySparkSQLEngine.scala vendored
View File

@ -17,11 +17,12 @@
package org.apache.kyuubi.engine.spark
import org.apache.curator.framework.CuratorFramework
import org.apache.kyuubi.Utils
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ha.HighAvailabilityConf.{HA_ZK_ACL_ENABLED, HA_ZK_NAMESPACE, HA_ZK_QUORUM}
import org.apache.kyuubi.ha.client.ServiceDiscovery
import org.apache.kyuubi.shade.curator.framework.CuratorFramework
import org.apache.kyuubi.zookeeper.{EmbeddedZookeeper, ZookeeperConf}
trait WithDiscoverySparkSQLEngine extends WithSparkSQLEngine {

6
externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala vendored
View File

@ -24,6 +24,9 @@ import scala.util.Random
import org.apache.hive.common.util.HiveVersionInfo
import org.apache.hive.service.cli.HiveSQLException
import org.apache.hive.service.rpc.thrift._
import org.apache.hive.service.rpc.thrift.TCLIService.Iface
import org.apache.hive.service.rpc.thrift.TOperationState._
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.types._
@ -32,9 +35,6 @@ import org.apache.kyuubi.engine.spark.WithSparkSQLEngine
import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim
import org.apache.kyuubi.operation.JDBCTests
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
import org.apache.kyuubi.shade.hive.service.rpc.thrift._
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TCLIService.Iface
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TOperationState._
class SparkOperationSuite extends WithSparkSQLEngine with JDBCTests {

2
externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/schema/RowSetSuite.scala vendored
View File

@ -24,13 +24,13 @@ import java.time.{Instant, LocalDate, ZoneId}
import scala.collection.JavaConverters._
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.CalendarInterval
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.schema.RowSet.toHiveString
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
class RowSetSuite extends KyuubiFunSuite {

2
externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/schema/SchemaHelperSuite.scala vendored
View File

@ -19,10 +19,10 @@ package org.apache.kyuubi.schema
import scala.collection.JavaConverters._
import org.apache.hive.service.rpc.thrift.{TCLIServiceConstants, TTypeId}
import org.apache.spark.sql.types._
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIServiceConstants, TTypeId}
class SchemaHelperSuite extends KyuubiFunSuite {

33
kyuubi-assembly/pom.xml
View File

@ -63,18 +63,6 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-shaded-curator</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-shaded-hive-service-rpc</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
@ -85,6 +73,27 @@
<artifactId>hadoop-client-runtime</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
</dependency>
<!-- for EmbeddedZkServer -->
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
</dependency>
<!-- default package with log4j -->
<dependency>
<groupId>org.slf4j</groupId>

11
kyuubi-common/pom.xml
View File

@ -33,12 +33,6 @@
<dependencies>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-shaded-hive-service-rpc</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@ -80,6 +74,11 @@
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-service-rpc</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>

2
kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiSQLException.scala
View File

@ -24,7 +24,7 @@ import java.sql.SQLException
import scala.annotation.tailrec
import scala.collection.JavaConverters._
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TStatus, TStatusCode}
import org.apache.hive.service.rpc.thrift.{TStatus, TStatusCode}
class KyuubiSQLException(msg: String, cause: Throwable) extends SQLException(msg, cause) {
/**

2
kyuubi-common/src/main/scala/org/apache/kyuubi/cli/Handle.scala
View File

@ -17,7 +17,7 @@
package org.apache.kyuubi.cli
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
import org.apache.hive.service.rpc.thrift.TProtocolVersion
trait Handle {
def identifier: HandleIdentifier

2
kyuubi-common/src/main/scala/org/apache/kyuubi/cli/HandleIdentifier.scala
View File

@ -20,7 +20,7 @@ package org.apache.kyuubi.cli
import java.nio.ByteBuffer
import java.util.{Objects, UUID}
import org.apache.kyuubi.shade.hive.service.rpc.thrift.THandleIdentifier
import org.apache.hive.service.rpc.thrift.THandleIdentifier
class HandleIdentifier(val publicId: UUID, val secretId: UUID) {

3
kyuubi-common/src/main/scala/org/apache/kyuubi/operation/AbstractOperation.scala
View File

@ -20,13 +20,14 @@ package org.apache.kyuubi.operation
import java.time.Duration
import java.util.concurrent.Future
import org.apache.hive.service.rpc.thrift.{TProtocolVersion, TRowSet, TTableSchema}
import org.apache.kyuubi.{KyuubiSQLException, Logging}
import org.apache.kyuubi.config.KyuubiConf.OPERATION_IDLE_TIMEOUT
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.operation.OperationType.OperationType
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TProtocolVersion, TRowSet, TTableSchema}
abstract class AbstractOperation(opType: OperationType, session: Session)
extends Operation with Logging {

2
kyuubi-common/src/main/scala/org/apache/kyuubi/operation/FetchOrientation.scala
View File

@ -17,7 +17,7 @@
package org.apache.kyuubi.operation
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TFetchOrientation
import org.apache.hive.service.rpc.thrift.TFetchOrientation
object FetchOrientation extends Enumeration {
type FetchOrientation = Value

3
kyuubi-common/src/main/scala/org/apache/kyuubi/operation/Operation.scala
View File

@ -19,10 +19,11 @@ package org.apache.kyuubi.operation
import java.util.concurrent.Future
import org.apache.hive.service.rpc.thrift.{TProtocolVersion, TRowSet, TTableSchema}
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TProtocolVersion, TRowSet, TTableSchema}
trait Operation {

3
kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationHandle.scala
View File

@ -21,9 +21,10 @@ import java.util.Objects
import scala.language.implicitConversions
import org.apache.hive.service.rpc.thrift.{TOperationHandle, TProtocolVersion}
import org.apache.kyuubi.cli.{Handle, HandleIdentifier}
import org.apache.kyuubi.operation.OperationType.OperationType
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TOperationHandle, TProtocolVersion}
case class OperationHandle(
identifier: HandleIdentifier,

3
kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationManager.scala
View File

@ -17,13 +17,14 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift._
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.operation.log.LogDivertAppender
import org.apache.kyuubi.service.AbstractService
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift._
/**
* The [[OperationManager]] manages all the operations during their lifecycle.

3
kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationState.scala
View File

@ -19,8 +19,9 @@ package org.apache.kyuubi.operation
import scala.language.implicitConversions
import org.apache.hive.service.rpc.thrift.TOperationState
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TOperationState
object OperationState extends Enumeration {
import TOperationState._

2
kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationType.scala
View File

@ -17,7 +17,7 @@
package org.apache.kyuubi.operation
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TOperationType
import org.apache.hive.service.rpc.thrift.TOperationType
object OperationType extends Enumeration {

3
kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/OperationLog.scala
View File

@ -22,10 +22,11 @@ import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path, Paths}
import org.apache.hive.service.rpc.thrift.{TColumn, TRow, TRowSet, TStringColumn}
import org.apache.kyuubi.{KyuubiSQLException, Logging, Utils}
import org.apache.kyuubi.operation.OperationHandle
import org.apache.kyuubi.session.SessionHandle
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TColumn, TRow, TRowSet, TStringColumn}
object OperationLog extends Logging {

3
kyuubi-common/src/main/scala/org/apache/kyuubi/service/AbstractBackendService.scala
View File

@ -21,11 +21,12 @@ import java.util.concurrent.{ExecutionException, TimeoutException, TimeUnit}
import scala.concurrent.CancellationException
import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion, TRowSet, TTableSchema}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.operation.{OperationHandle, OperationStatus}
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.session.SessionHandle
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion, TRowSet, TTableSchema}
/**
* A Shorthand for implementing [[BackendService]]s

3
kyuubi-common/src/main/scala/org/apache/kyuubi/service/BackendService.scala
View File

@ -17,10 +17,11 @@
package org.apache.kyuubi.service
import org.apache.hive.service.rpc.thrift._
import org.apache.kyuubi.operation.{OperationHandle, OperationStatus}
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.session.{SessionHandle, SessionManager}
import org.apache.kyuubi.shade.hive.service.rpc.thrift._
/**
* A [[BackendService]] in Kyuubi architecture is responsible for talking to the SQL engine

8
kyuubi-common/src/main/scala/org/apache/kyuubi/service/FrontendService.scala
View File

@ -24,16 +24,16 @@ import scala.collection.JavaConverters._
import scala.language.implicitConversions
import org.apache.hadoop.conf.Configuration
import org.apache.hive.service.rpc.thrift._
import org.apache.thrift.protocol.{TBinaryProtocol, TProtocol}
import org.apache.thrift.server.{ServerContext, TServer, TServerEventHandler, TThreadPoolServer}
import org.apache.thrift.transport.{TServerSocket, TTransport}
import org.apache.kyuubi.{KyuubiException, KyuubiSQLException, Logging}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.operation.{FetchOrientation, OperationHandle}
import org.apache.kyuubi.service.authentication.KyuubiAuthenticationFactory
import org.apache.kyuubi.session.SessionHandle
import org.apache.kyuubi.shade.hive.service.rpc.thrift._
import org.apache.kyuubi.shade.thrift.protocol.{TBinaryProtocol, TProtocol}
import org.apache.kyuubi.shade.thrift.server.{ServerContext, TServer, TServerEventHandler, TThreadPoolServer}
import org.apache.kyuubi.shade.thrift.transport.{TServerSocket, TTransport}
import org.apache.kyuubi.util.{ExecutorPoolCaptureOom, KyuubiHadoopUtils, NamedThreadFactory}
class FrontendService private (name: String, be: BackendService, oomHook: Runnable)

6
kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/FEServiceProcessorFactory.scala
View File

@ -17,9 +17,9 @@
package org.apache.kyuubi.service.authentication
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TCLIService.{Iface, Processor}
import org.apache.kyuubi.shade.thrift.{TProcessor, TProcessorFactory}
import org.apache.kyuubi.shade.thrift.transport.TTransport
import org.apache.hive.service.rpc.thrift.TCLIService.{Iface, Processor}
import org.apache.thrift.{TProcessor, TProcessorFactory}
import org.apache.thrift.transport.TTransport
private[authentication]
case class FEServiceProcessorFactory(saslServer: HadoopThriftAuthBridgeServer, service: Iface)

6
kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/HadoopThriftAuthBridgeServer.scala
View File

@ -28,11 +28,11 @@ import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.security.{SaslRpcServer, UserGroupInformation}
import org.apache.hadoop.security.SaslRpcServer.AuthMethod
import org.apache.hadoop.security.token.SecretManager.InvalidToken
import org.apache.thrift.{TException, TProcessor}
import org.apache.thrift.protocol.TProtocol
import org.apache.thrift.transport._
import org.apache.kyuubi.Logging
import org.apache.kyuubi.shade.thrift.{TException, TProcessor}
import org.apache.kyuubi.shade.thrift.protocol.TProtocol
import org.apache.kyuubi.shade.thrift.transport._
class HadoopThriftAuthBridgeServer(secretMgr: KyuubiDelegationTokenManager) {
import HadoopThriftAuthBridgeServer._

6
kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala
View File

@ -25,14 +25,14 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.security.authentication.util.KerberosName
import org.apache.hadoop.security.authorize.ProxyUsers
import org.apache.hive.service.rpc.thrift.TCLIService.Iface
import org.apache.thrift.TProcessorFactory
import org.apache.thrift.transport.{TTransportException, TTransportFactory}
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.service.authentication.AuthTypes._
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TCLIService.Iface
import org.apache.kyuubi.shade.thrift.TProcessorFactory
import org.apache.kyuubi.shade.thrift.transport.{TTransportException, TTransportFactory}
class KyuubiAuthenticationFactory(conf: KyuubiConf) {

7
kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLHelper.scala
View File

@ -22,12 +22,13 @@ import java.util.Collections
import javax.security.auth.callback.{Callback, CallbackHandler, NameCallback, PasswordCallback, UnsupportedCallbackException}
import javax.security.sasl.AuthorizeCallback
import org.apache.hive.service.rpc.thrift.TCLIService.Iface
import org.apache.thrift.{TProcessor, TProcessorFactory}
import org.apache.thrift.transport.{TSaslClientTransport, TSaslServerTransport, TTransport, TTransportFactory}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.service.authentication.AuthMethods.AuthMethod
import org.apache.kyuubi.service.authentication.PlainSASLServer.SaslPlainProvider
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TCLIService.Iface
import org.apache.kyuubi.shade.thrift.{TProcessor, TProcessorFactory}
import org.apache.kyuubi.shade.thrift.transport.{TSaslClientTransport, TSaslServerTransport, TTransport, TTransportFactory}
object PlainSASLHelper {

9
kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/TSetIpAddressProcessor.scala
View File

@ -17,11 +17,12 @@
package org.apache.kyuubi.service.authentication
import org.apache.hive.service.rpc.thrift.TCLIService.{Iface, Processor}
import org.apache.thrift.TException
import org.apache.thrift.protocol.TProtocol
import org.apache.thrift.transport.{TSaslClientTransport, TSaslServerTransport, TSocket, TTransport}
import org.apache.kyuubi.Logging
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TCLIService.{Iface, Processor}
import org.apache.kyuubi.shade.thrift.TException
import org.apache.kyuubi.shade.thrift.protocol.TProtocol
import org.apache.kyuubi.shade.thrift.transport.{TSaslClientTransport, TSaslServerTransport, TSocket, TTransport}
class TSetIpAddressProcessor[I <: Iface](
iface: Iface) extends Processor[Iface](iface) with Logging {

3
kyuubi-common/src/main/scala/org/apache/kyuubi/session/AbstractSession.scala
View File

@ -19,11 +19,12 @@ package org.apache.kyuubi.session
import scala.collection.JavaConverters._
import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion, TRowSet, TTableSchema}
import org.apache.kyuubi.{KyuubiSQLException, Logging}
import org.apache.kyuubi.operation.{Operation, OperationHandle}
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion, TRowSet, TTableSchema}
abstract class AbstractSession(
val protocol: TProtocolVersion,

3
kyuubi-common/src/main/scala/org/apache/kyuubi/session/Session.scala
View File

@ -17,9 +17,10 @@
package org.apache.kyuubi.session
import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion, TRowSet, TTableSchema}
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.operation.OperationHandle
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion, TRowSet, TTableSchema}
trait Session {

3
kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionHandle.scala
View File

@ -19,8 +19,9 @@ package org.apache.kyuubi.session
import java.util.Objects
import org.apache.hive.service.rpc.thrift.{TProtocolVersion, TSessionHandle}
import org.apache.kyuubi.cli.{Handle, HandleIdentifier}
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TProtocolVersion, TSessionHandle}
case class SessionHandle(
identifier: HandleIdentifier,

3
kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala
View File

@ -21,12 +21,13 @@ import java.util.concurrent.{ConcurrentHashMap, Future, ThreadPoolExecutor, Time
import scala.collection.JavaConverters._
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.operation.OperationManager
import org.apache.kyuubi.service.CompositeService
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
import org.apache.kyuubi.util.ThreadUtils
/**

3
kyuubi-common/src/main/scala/org/apache/kyuubi/util/ThriftUtils.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.util
import org.apache.hive.service.rpc.thrift.{TRow, TRowSet, TStatus, TStatusCode}
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TRow, TRowSet, TStatus, TStatusCode}
object ThriftUtils {

2
kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiSQLExceptionSuite.scala
View File

@ -19,7 +19,7 @@ package org.apache.kyuubi
import java.lang.reflect.{InvocationTargetException, UndeclaredThrowableException}
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TStatusCode
import org.apache.hive.service.rpc.thrift.TStatusCode
class KyuubiSQLExceptionSuite extends KyuubiFunSuite {

3
kyuubi-common/src/test/scala/org/apache/kyuubi/operation/FetchOrientationSuite.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.TFetchOrientation
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TFetchOrientation
class FetchOrientationSuite extends KyuubiFunSuite {

7
kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala
View File

@ -20,11 +20,12 @@ package org.apache.kyuubi.operation
import java.sql.{DriverManager, ResultSet, Statement}
import java.util.Locale
import org.apache.hive.service.rpc.thrift.{TCLIService, TCloseSessionReq, TOpenSessionReq, TSessionHandle}
import org.apache.thrift.protocol.TBinaryProtocol
import org.apache.thrift.transport.TSocket
import org.apache.kyuubi.{KyuubiFunSuite, Utils}
import org.apache.kyuubi.service.authentication.PlainSASLHelper
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TCloseSessionReq, TOpenSessionReq, TSessionHandle}
import org.apache.kyuubi.shade.thrift.protocol.TBinaryProtocol
import org.apache.kyuubi.shade.thrift.transport.TSocket
trait JDBCTestUtils extends KyuubiFunSuite {

2
kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala
View File

@ -21,7 +21,7 @@ import java.sql.{Date, SQLException, SQLTimeoutException, Timestamp}
import scala.collection.JavaConverters._
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TExecuteStatementReq, TFetchResultsReq, TOpenSessionReq, TStatusCode}
import org.apache.hive.service.rpc.thrift.{TExecuteStatementReq, TFetchResultsReq, TOpenSessionReq, TStatusCode}
trait JDBCTests extends BasicJDBCTests {
test("execute statement - select null") {

3
kyuubi-common/src/test/scala/org/apache/kyuubi/operation/NoopOperation.scala
View File

@ -21,12 +21,13 @@ import java.nio.ByteBuffer
import scala.collection.JavaConverters._
import org.apache.hive.service.rpc.thrift.{TColumn, TColumnDesc, TPrimitiveTypeEntry, TRowSet, TStringColumn, TTableSchema, TTypeDesc, TTypeEntry, TTypeId}
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.operation.OperationType.OperationType
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift._
import org.apache.kyuubi.util.ThriftUtils
class NoopOperation(typ: OperationType, session: Session, shouldFail: Boolean = false)

3
kyuubi-common/src/test/scala/org/apache/kyuubi/operation/NoopOperationManager.scala
View File

@ -17,9 +17,10 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.TRowSet
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TRowSet
import org.apache.kyuubi.util.ThriftUtils
class NoopOperationManager extends OperationManager("noop") {

5
kyuubi-common/src/test/scala/org/apache/kyuubi/operation/OperationHandleSuite.scala
View File

@ -17,11 +17,12 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.hive.service.rpc.thrift.TProtocolVersion._
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.cli.HandleIdentifier
import org.apache.kyuubi.operation.OperationType._
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion._
class OperationHandleSuite extends KyuubiFunSuite {

5
kyuubi-common/src/test/scala/org/apache/kyuubi/operation/OperationStateSuite.scala
View File

@ -17,9 +17,10 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.TOperationState
import org.apache.hive.service.rpc.thrift.TOperationState._
import org.apache.kyuubi.{KyuubiFunSuite, KyuubiSQLException}
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TOperationState
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TOperationState._
class OperationStateSuite extends KyuubiFunSuite {
import OperationState._

3
kyuubi-common/src/test/scala/org/apache/kyuubi/operation/OperationTypeSuite.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.TOperationType
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TOperationType
class OperationTypeSuite extends KyuubiFunSuite {

3
kyuubi-common/src/test/scala/org/apache/kyuubi/operation/log/OperationLogSuite.scala
View File

@ -21,10 +21,11 @@ import java.nio.file.{Files, Paths}
import scala.collection.JavaConverters._
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.kyuubi.{KyuubiFunSuite, KyuubiSQLException}
import org.apache.kyuubi.operation.{OperationHandle, OperationType}
import org.apache.kyuubi.session.SessionHandle
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
class OperationLogSuite extends KyuubiFunSuite {

7
kyuubi-common/src/test/scala/org/apache/kyuubi/service/FrontendServiceSuite.scala
View File

@ -22,15 +22,16 @@ import java.util
import scala.collection.JavaConverters._
import org.apache.hive.service.rpc.thrift._
import org.apache.thrift.protocol.TBinaryProtocol
import org.apache.thrift.transport.TSocket
import org.apache.kyuubi.{KyuubiFunSuite, KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.operation.{OperationHandle, OperationType}
import org.apache.kyuubi.service.FrontendService.{FeServiceServerContext, SERVER_VERSION}
import org.apache.kyuubi.service.authentication.PlainSASLHelper
import org.apache.kyuubi.session.SessionHandle
import org.apache.kyuubi.shade.hive.service.rpc.thrift._
import org.apache.kyuubi.shade.thrift.protocol.TBinaryProtocol
import org.apache.kyuubi.shade.thrift.transport.TSocket
class FrontendServiceSuite extends KyuubiFunSuite {

3
kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLHelperSuite.scala
View File

@ -19,11 +19,12 @@ package org.apache.kyuubi.service.authentication
import java.security.Security
import org.apache.thrift.transport.{TSaslServerTransport, TSocket}
import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.service.{FrontendService, NoopServer}
import org.apache.kyuubi.service.authentication.PlainSASLServer.SaslPlainProvider
import org.apache.kyuubi.shade.thrift.transport.{TSaslServerTransport, TSocket}
class PlainSASLHelperSuite extends KyuubiFunSuite {

2
kyuubi-common/src/test/scala/org/apache/kyuubi/session/NoopSessionImpl.scala
View File

@ -17,7 +17,7 @@
package org.apache.kyuubi.session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
import org.apache.hive.service.rpc.thrift.TProtocolVersion
class NoopSessionImpl(
protocol: TProtocolVersion,

3
kyuubi-common/src/test/scala/org/apache/kyuubi/session/NoopSessionManager.scala
View File

@ -17,9 +17,10 @@
package org.apache.kyuubi.session
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.operation.{NoopOperationManager, OperationManager}
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
class NoopSessionManager extends SessionManager("noop") {
override val operationManager: OperationManager = new NoopOperationManager()

3
kyuubi-common/src/test/scala/org/apache/kyuubi/session/SessionHandleSuite.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.session
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
class SessionHandleSuite extends KyuubiFunSuite {
test("SessionHandle") {

2
kyuubi-common/src/test/scala/org/apache/kyuubi/session/SessionManagerSuite.scala
View File

@ -19,12 +19,12 @@ package org.apache.kyuubi.session
import java.time.Duration
import org.apache.hive.service.rpc.thrift._
import org.scalatest.concurrent.Eventually
import org.scalatest.time.{Seconds, Span}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.service.FrontendServiceSuite
import org.apache.kyuubi.shade.hive.service.rpc.thrift._
class SessionManagerSuite extends FrontendServiceSuite with Eventually {

23
kyuubi-ctl/pom.xml
View File

@ -42,11 +42,6 @@
<artifactId>kyuubi-ha</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-shaded-curator</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@ -60,11 +55,29 @@
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
<!-- Begin: for EmbeddedZkServer -->
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<scope>provided</scope>
</dependency>
<!-- End: for EmbeddedZkServer -->
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-common</artifactId>

10
kyuubi-ha/pom.xml
View File

@ -39,9 +39,13 @@
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-shaded-curator</artifactId>
<version>${project.version}</version>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
</dependency>
<dependency>

16
kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala
View File

@ -25,6 +25,12 @@ import javax.security.auth.login.Configuration
import scala.collection.JavaConverters._
import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
import org.apache.curator.framework.recipes.nodes.PersistentEphemeralNode
import org.apache.curator.framework.state.{ConnectionState, ConnectionStateListener}
import org.apache.curator.framework.state.ConnectionState.{CONNECTED, LOST, RECONNECTED}
import org.apache.curator.retry._
import org.apache.curator.utils.ZKPaths
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.JaasConfiguration
import org.apache.zookeeper.{WatchedEvent, Watcher}
@ -37,12 +43,6 @@ import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ha.HighAvailabilityConf._
import org.apache.kyuubi.ha.client.ServiceDiscovery._
import org.apache.kyuubi.service.{AbstractService, Serverable}
import org.apache.kyuubi.shade.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
import org.apache.kyuubi.shade.curator.framework.recipes.nodes.{PersistentEphemeralNode, PersistentNode}
import org.apache.kyuubi.shade.curator.framework.state.{ConnectionState, ConnectionStateListener}
import org.apache.kyuubi.shade.curator.framework.state.ConnectionState.{CONNECTED, LOST, RECONNECTED}
import org.apache.kyuubi.shade.curator.retry._
import org.apache.kyuubi.shade.curator.utils.ZKPaths
import org.apache.kyuubi.util.{KyuubiHadoopUtils, ThreadUtils}
/**
@ -59,14 +59,14 @@ abstract class ServiceDiscovery private (
this(classOf[ServiceDiscovery].getSimpleName, server)
private var _zkClient: CuratorFramework = _
private var _serviceNode: PersistentNode = _
private var _serviceNode: PersistentEphemeralNode = _
/**
* a pre-defined namespace used to publish the instance of the associate service
*/
private var _namespace: String = _
def zkClient: CuratorFramework = _zkClient
def serviceNode: PersistentNode = _serviceNode
def serviceNode: PersistentEphemeralNode = _serviceNode
def namespace: String = _namespace
override def initialize(conf: KyuubiConf): Unit = {

2
kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ZooKeeperACLProvider.scala
View File

@ -17,12 +17,12 @@
package org.apache.kyuubi.ha.client
import org.apache.curator.framework.api.ACLProvider
import org.apache.zookeeper.ZooDefs
import org.apache.zookeeper.data.ACL
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ha.HighAvailabilityConf
import org.apache.kyuubi.shade.curator.framework.api.ACLProvider
class ZooKeeperACLProvider(conf: KyuubiConf) extends ACLProvider {

12
kyuubi-main/pom.xml
View File

@ -82,12 +82,6 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-service-rpc</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-spark-sql-engine</artifactId>
@ -133,6 +127,12 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>${iceberg.name}</artifactId>

3
kyuubi-main/src/main/scala/org/apache/kyuubi/engine/SQLEngineAppName.scala
View File

@ -17,9 +17,10 @@
package org.apache.kyuubi.engine
import org.apache.curator.utils.ZKPaths
import org.apache.kyuubi.engine.ShareLevel.{CONNECTION, ShareLevel}
import org.apache.kyuubi.session.SessionHandle
import org.apache.kyuubi.shade.curator.utils.ZKPaths
case class SQLEngineAppName(
sharedLevel: ShareLevel, user: String, sessionId: String) {

5
kyuubi-main/src/main/scala/org/apache/kyuubi/operation/ExecuteStatement.scala
View File

@ -19,13 +19,14 @@ package org.apache.kyuubi.operation
import scala.collection.JavaConverters._
import org.apache.hive.service.rpc.thrift.{TCLIService, TExecuteStatementReq, TFetchOrientation, TFetchResultsReq, TGetOperationStatusReq, TSessionHandle}
import org.apache.hive.service.rpc.thrift.TOperationState._
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.metrics.MetricsConstants._
import org.apache.kyuubi.metrics.MetricsSystem
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TExecuteStatementReq, TFetchOrientation, TFetchResultsReq, TGetOperationStatusReq, TSessionHandle}
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TOperationState._
class ExecuteStatement(
session: Session,

3
kyuubi-main/src/main/scala/org/apache/kyuubi/operation/GetCatalogs.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.{TCLIService, TGetCatalogsReq, TSessionHandle}
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TGetCatalogsReq, TSessionHandle}
class GetCatalogs(
session: Session,

3
kyuubi-main/src/main/scala/org/apache/kyuubi/operation/GetColumns.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.{TCLIService, TGetColumnsReq, TSessionHandle}
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TGetColumnsReq, TSessionHandle}
class GetColumns(
session: Session,

3
kyuubi-main/src/main/scala/org/apache/kyuubi/operation/GetFunctions.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.{TCLIService, TGetFunctionsReq, TSessionHandle}
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TGetFunctionsReq, TSessionHandle}
class GetFunctions(
session: Session,

3
kyuubi-main/src/main/scala/org/apache/kyuubi/operation/GetSchemas.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.{TCLIService, TGetSchemasReq, TSessionHandle}
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TGetSchemasReq, TSessionHandle}
class GetSchemas(
session: Session,

3
kyuubi-main/src/main/scala/org/apache/kyuubi/operation/GetTableTypes.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.{TCLIService, TGetTableTypesReq, TSessionHandle}
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TGetTableTypesReq, TSessionHandle}
class GetTableTypes(
session: Session,

3
kyuubi-main/src/main/scala/org/apache/kyuubi/operation/GetTables.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.{TCLIService, TGetTablesReq, TSessionHandle}
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TGetTablesReq, TSessionHandle}
class GetTables(
session: Session,

3
kyuubi-main/src/main/scala/org/apache/kyuubi/operation/GetTypeInfo.scala
View File

@ -17,8 +17,9 @@
package org.apache.kyuubi.operation
import org.apache.hive.service.rpc.thrift.{TCLIService, TGetTypeInfoReq, TSessionHandle}
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TGetTypeInfoReq, TSessionHandle}
class GetTypeInfo(
session: Session,

4
kyuubi-main/src/main/scala/org/apache/kyuubi/operation/KyuubiOperation.scala
View File

@ -19,6 +19,8 @@ package org.apache.kyuubi.operation
import com.codahale.metrics.MetricRegistry
import org.apache.commons.lang3.StringUtils
import org.apache.hive.service.rpc.thrift._
import org.apache.thrift.transport.TTransportException
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.metrics.MetricsConstants.STATEMENT_FAIL
@ -26,8 +28,6 @@ import org.apache.kyuubi.metrics.MetricsSystem
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.operation.OperationType.OperationType
import org.apache.kyuubi.session.Session
import org.apache.kyuubi.shade.hive.service.rpc.thrift._
import org.apache.kyuubi.shade.thrift.transport.TTransportException
import org.apache.kyuubi.util.ThriftUtils
abstract class KyuubiOperation(

3
kyuubi-main/src/main/scala/org/apache/kyuubi/operation/KyuubiOperationManager.scala
View File

@ -19,12 +19,13 @@ package org.apache.kyuubi.operation
import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
import org.apache.hive.service.rpc.thrift.{TCLIService, TFetchResultsReq, TRowSet, TSessionHandle}
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.OPERATION_QUERY_TIMEOUT
import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation
import org.apache.kyuubi.session.{Session, SessionHandle}
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TFetchResultsReq, TRowSet, TSessionHandle}
import org.apache.kyuubi.util.ThriftUtils
class KyuubiOperationManager private (name: String) extends OperationManager(name) {

8
kyuubi-main/src/main/scala/org/apache/kyuubi/session/KyuubiSessionImpl.scala
View File

@ -23,6 +23,10 @@ import java.util.concurrent.TimeUnit
import scala.collection.JavaConverters._
import com.codahale.metrics.MetricRegistry
import org.apache.hive.service.rpc.thrift.{TCLIService, TCloseSessionReq, TOpenSessionReq, TProtocolVersion, TSessionHandle}
import org.apache.thrift.TException
import org.apache.thrift.protocol.TBinaryProtocol
import org.apache.thrift.transport.{TSocket, TTransport}
import org.apache.kyuubi.{KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiConf
@ -35,10 +39,6 @@ import org.apache.kyuubi.ha.client.ServiceDiscovery._
import org.apache.kyuubi.metrics.MetricsConstants._
import org.apache.kyuubi.metrics.MetricsSystem
import org.apache.kyuubi.service.authentication.PlainSASLHelper
import org.apache.kyuubi.shade.hive.service.rpc.thrift.{TCLIService, TCloseSessionReq, TOpenSessionReq, TProtocolVersion, TSessionHandle}
import org.apache.kyuubi.shade.thrift.TException
import org.apache.kyuubi.shade.thrift.protocol.TBinaryProtocol
import org.apache.kyuubi.shade.thrift.transport.{TSocket, TTransport}
import org.apache.kyuubi.util.ThriftUtils
class KyuubiSessionImpl(

2
kyuubi-main/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala
View File

@ -18,6 +18,7 @@
package org.apache.kyuubi.session
import com.codahale.metrics.MetricRegistry
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.config.KyuubiConf
@ -25,7 +26,6 @@ import org.apache.kyuubi.ha.client.ServiceDiscovery
import org.apache.kyuubi.metrics.MetricsConstants._
import org.apache.kyuubi.metrics.MetricsSystem
import org.apache.kyuubi.operation.KyuubiOperationManager
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
class KyuubiSessionManager private (name: String) extends SessionManager(name) {

5
kyuubi-main/src/test/scala/org/apache/kyuubi/engine/SQLEngineAppNameSuite.scala
View File

@ -17,10 +17,11 @@
package org.apache.kyuubi.engine
import org.apache.curator.utils.ZKPaths
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.kyuubi.{KyuubiFunSuite, Utils}
import org.apache.kyuubi.session.SessionHandle
import org.apache.kyuubi.shade.curator.utils.ZKPaths
import org.apache.kyuubi.shade.hive.service.rpc.thrift.TProtocolVersion
class SQLEngineAppNameSuite extends KyuubiFunSuite {
import ShareLevel._

132
kyuubi-shaded/kyuubi-shaded-curator/pom.xml
View File

@ -1,132 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>kyuubi-shaded</artifactId>
<groupId>org.apache.kyuubi</groupId>
<version>1.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>kyuubi-shaded-curator</artifactId>
<name>Kyuubi Project Shade Apache Curator</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</exclusion>
<exclusion>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<configuration>
<artifactSet>
<includes>
<include>org.apache.curator:curator-client</include>
<include>org.apache.curator:curator-framework</include>
<include>org.apache.curator:curator-recipes</include>
<include>org.apache.curator:curator-test</include>
<include>org.javassist:javassist</include>
</includes>
</artifactSet>
<relocations>
<relocation>
<pattern>org.apache.curator</pattern>
<shadedPattern>${kyuubi.shade.packageName}.curator</shadedPattern>
<includes>
<include>org.apache.curator.**</include>
</includes>
</relocation>
<relocation>
<pattern>org.javassist</pattern>
<shadedPattern>${kyuubi.shade.packageName}.javassist</shadedPattern>
<includes>
<include>org.javassist.**</include>
</includes>
</relocation>
</relocations>
</configuration>
</plugin>
</plugins>
</build>
</project>

96
kyuubi-shaded/kyuubi-shaded-hive-service-rpc/pom.xml
View File

@ -1,96 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>kyuubi-shaded</artifactId>
<groupId>org.apache.kyuubi</groupId>
<version>1.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>kyuubi-shaded-hive-service-rpc</artifactId>
<name>Kyuubi Project Shade Hive Service Rpc</name>
<packaging>jar</packaging>
<dependencies>
<!--
because of THRIFT-4805, we don't upgrade to libthrift:0.12.0,
because of THRIFT-5274, we don't upgrade to libthrift:0.13.0,
so just keep hive-service-rpc:2.3.7 transitive dependency libthrift:0.9.3
-->
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-service-rpc</artifactId>
<version>${hive.service.rpc.version}</version>
<exclusions>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<configuration>
<artifactSet>
<includes>
<include>org.apache.hive:hive-service-rpc</include>
<include>org.apache.thrift:libthrift</include>
<include>org.apache.thrift:libfb303</include>
</includes>
</artifactSet>
<relocations>
<relocation>
<pattern>org.apache.hive</pattern>
<shadedPattern>${kyuubi.shade.packageName}.hive</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.thrift</pattern>
<shadedPattern>${kyuubi.shade.packageName}.thrift</shadedPattern>
</relocation>
</relocations>
</configuration>
</plugin>
</plugins>
</build>
</project>

62
kyuubi-shaded/pom.xml
View File

@ -1,62 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>1.2.0-SNAPSHOT</version>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-shaded</artifactId>
<name>Kyuubi Project Shaded Dependencies</name>
<packaging>pom</packaging>
<properties>
<curator.version>2.12.0</curator.version>
<hive.service.rpc.version>2.3.7</hive.service.rpc.version>
<!-- Package to use when relocating shaded classes. -->
<kyuubi.shade.packageName>org.apache.kyuubi.shade</kyuubi.shade.packageName>
<scalastyle.skip>true</scalastyle.skip>
</properties>
<modules>
<module>kyuubi-shaded-curator</module>
<module>kyuubi-shaded-hive-service-rpc</module>
</modules>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<configuration>
<shadedArtifactAttached>false</shadedArtifactAttached>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

24
kyuubi-zookeeper/pom.xml
View File

@ -37,17 +37,18 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-shaded-curator</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
<!-- Begin: for EmbeddedZkServer -->
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
</dependency>
<!-- End: for EmbeddedZkServer -->
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-common</artifactId>
@ -55,6 +56,17 @@
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

3
kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala
View File

@ -21,9 +21,10 @@ import java.io.File
import scala.collection.JavaConverters._
import org.apache.curator.test.{InstanceSpec, TestingServer}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.service.{AbstractService, ServiceState}
import org.apache.kyuubi.shade.curator.test.{InstanceSpec, TestingServer}
import org.apache.kyuubi.zookeeper.ZookeeperConf._
class EmbeddedZookeeper extends AbstractService("EmbeddedZookeeper") {

7
kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala
View File

@ -17,12 +17,13 @@
package org.apache.kyuubi.zookeeper
import org.apache.curator.framework.CuratorFrameworkFactory
import org.apache.curator.framework.imps.CuratorFrameworkState
import org.apache.curator.retry.ExponentialBackoffRetry
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.service.ServiceState._
import org.apache.kyuubi.shade.curator.framework.CuratorFrameworkFactory
import org.apache.kyuubi.shade.curator.framework.imps.CuratorFrameworkState
import org.apache.kyuubi.shade.curator.retry.ExponentialBackoffRetry
class EmbeddedZookeeperSuite extends KyuubiFunSuite {

71
pom.xml
View File

@ -67,6 +67,7 @@
<commons-codec.version>1.15</commons-codec.version>
<commons-collections.version>3.2.2</commons-collections.version>
<commons-lang3.version>3.10</commons-lang3.version>
<curator.version>2.12.0</curator.version>
<guava.version>30.1-jre</guava.version>
<hadoop.version>3.2.2</hadoop.version>
<hadoop.binary.version>2.7</hadoop.binary.version>
@ -814,6 +815,70 @@
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</exclusion>
<exclusion>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@ -1375,11 +1440,5 @@
<spark.archive.download.skip>true</spark.archive.download.skip>
</properties>
</profile>
<profile>
<id>kyuubi-shaded</id>
<modules>
<module>kyuubi-shaded</module>
</modules>
</profile>
</profiles>
</project>