apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #1542] Move FetchIterator into kyuubi-common for multi-engine support

Browse Source 
…

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #1543 from yanghua/KYUUBI-1542.

Closes #1542

1ca1c08a [yanghua] Move FetchIteratorSuite into kyuubi-common module
1307d886 [yanghua] [KYUUBI #1542] Move FetchIterator into kyuubi-common for multi-engine support

Authored-by: yanghua <yanghua1127@gmail.com>
Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
yanghua 2021-12-10 16:29:05 +08:00 committed by Cheng Pan
parent 3942cf3030
commit 6045104092
No known key found for this signature in database
GPG Key ID: 8001952629BCC75D
13 changed files with 17 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala vendored
View File

@ -23,9 +23,8 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.types.StructType
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.engine.spark.ArrayFetchIterator
import org.apache.kyuubi.engine.spark.repl.KyuubiSparkILoop
import org.apache.kyuubi.operation.OperationType
import org.apache.kyuubi.operation.{ArrayFetchIterator, OperationType}
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.session.Session

4
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala vendored
View File

@ -27,9 +27,9 @@ import org.apache.spark.sql.types._
import org.apache.kyuubi.{KyuubiSQLException, Logging}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.engine.spark.{ArrayFetchIterator, IterableFetchIterator, KyuubiSparkUtil}
import org.apache.kyuubi.engine.spark.KyuubiSparkUtil
import org.apache.kyuubi.engine.spark.events.{EventLoggingService, SparkStatementEvent}
import org.apache.kyuubi.operation.{OperationState, OperationType}
import org.apache.kyuubi.operation.{ArrayFetchIterator, IterableFetchIterator, OperationState, OperationType}
import org.apache.kyuubi.operation.OperationState.OperationState
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.session.Session

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCatalogs.scala vendored
View File

@ -19,9 +19,8 @@ package org.apache.kyuubi.engine.spark.operation
import org.apache.spark.sql.types.StructType
import org.apache.kyuubi.engine.spark.IterableFetchIterator
import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim
import org.apache.kyuubi.operation.OperationType
import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType}
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.TABLE_CAT
import org.apache.kyuubi.session.Session

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetColumns.scala vendored
View File

@ -19,9 +19,8 @@ package org.apache.kyuubi.engine.spark.operation
import org.apache.spark.sql.types._
import org.apache.kyuubi.engine.spark.IterableFetchIterator
import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim
import org.apache.kyuubi.operation.OperationType
import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType}
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
import org.apache.kyuubi.session.Session

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetFunctions.scala vendored
View File

@ -22,8 +22,7 @@ import java.sql.DatabaseMetaData
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.StructType
import org.apache.kyuubi.engine.spark.IterableFetchIterator
import org.apache.kyuubi.operation.OperationType
import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType}
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
import org.apache.kyuubi.session.Session

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetSchemas.scala vendored
View File

@ -19,9 +19,8 @@ package org.apache.kyuubi.engine.spark.operation
import org.apache.spark.sql.types.StructType
import org.apache.kyuubi.engine.spark.IterableFetchIterator
import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim
import org.apache.kyuubi.operation.OperationType
import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType}
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
import org.apache.kyuubi.session.Session

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTableTypes.scala vendored
View File

@ -20,9 +20,8 @@ package org.apache.kyuubi.engine.spark.operation
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.StructType
import org.apache.kyuubi.engine.spark.IterableFetchIterator
import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim
import org.apache.kyuubi.operation.OperationType
import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType}
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
import org.apache.kyuubi.session.Session

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala vendored
View File

@ -19,9 +19,8 @@ package org.apache.kyuubi.engine.spark.operation
import org.apache.spark.sql.types.StructType
import org.apache.kyuubi.engine.spark.IterableFetchIterator
import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim
import org.apache.kyuubi.operation.OperationType
import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType}
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
import org.apache.kyuubi.session.Session

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTypeInfo.scala vendored
View File

@ -22,8 +22,7 @@ import java.sql.Types._
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.StructType
import org.apache.kyuubi.engine.spark.IterableFetchIterator
import org.apache.kyuubi.operation.OperationType
import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType}
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
import org.apache.kyuubi.session.Session

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala vendored
View File

@ -22,8 +22,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.types.StructType
import org.apache.kyuubi.config.KyuubiConf.OperationModes.{ANALYZE, OperationMode, OPTIMIZE, PARSE}
import org.apache.kyuubi.engine.spark.{ArrayFetchIterator, IterableFetchIterator}
import org.apache.kyuubi.operation.OperationType
import org.apache.kyuubi.operation.{ArrayFetchIterator, IterableFetchIterator, OperationType}
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.session.Session

3
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala vendored
View File

@ -26,10 +26,9 @@ import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import org.apache.spark.sql.types.StructType
import org.apache.kyuubi.{KyuubiSQLException, Utils}
import org.apache.kyuubi.engine.spark.FetchIterator
import org.apache.kyuubi.engine.spark.operation.SparkOperation.TIMEZONE_KEY
import org.apache.kyuubi.engine.spark.session.SparkSessionImpl
import org.apache.kyuubi.operation.{AbstractOperation, OperationState}
import org.apache.kyuubi.operation.{AbstractOperation, FetchIterator, OperationState}
import org.apache.kyuubi.operation.FetchOrientation._
import org.apache.kyuubi.operation.OperationState.OperationState
import org.apache.kyuubi.operation.OperationType.OperationType

8
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/FetchIterator.scala → kyuubi-common/src/main/scala/org/apache/kyuubi/operation/FetchIterator.scala
View File

@ -15,12 +15,12 @@
* limitations under the License.
*/
package org.apache.kyuubi.engine.spark
package org.apache.kyuubi.operation
/**
* Borrowed from Apache Spark, see SPARK-33655
*/
sealed private[engine] trait FetchIterator[A] extends Iterator[A] {
sealed trait FetchIterator[A] extends Iterator[A] {
/**
* Begin a fetch block, forward from the current position.
@ -50,7 +50,7 @@ sealed private[engine] trait FetchIterator[A] extends Iterator[A] {
def getPosition: Long
}
private[engine] class ArrayFetchIterator[A](src: Array[A]) extends FetchIterator[A] {
class ArrayFetchIterator[A](src: Array[A]) extends FetchIterator[A] {
private var fetchStart: Long = 0
private var position: Long = 0
@ -74,7 +74,7 @@ private[engine] class ArrayFetchIterator[A](src: Array[A]) extends FetchIterator
}
}
private[engine] class IterableFetchIterator[A](iterable: Iterable[A]) extends FetchIterator[A] {
class IterableFetchIterator[A](iterable: Iterable[A]) extends FetchIterator[A] {
private var iter: Iterator[A] = iterable.iterator
private var fetchStart: Long = 0

2
externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/FetchIteratorSuite.scala → kyuubi-common/src/test/scala/org/apache/kyuubi/operation/FetchIteratorSuite.scala
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.kyuubi.engine.spark
package org.apache.kyuubi.operation
import org.apache.kyuubi.KyuubiFunSuite