From 604510409282ee40eebe00504128595284ecc34e Mon Sep 17 00:00:00 2001 From: yanghua Date: Fri, 10 Dec 2021 16:29:05 +0800 Subject: [PATCH] [KYUUBI #1542] Move FetchIterator into kyuubi-common for multi-engine support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … ### _Why are the changes needed?_ ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1543 from yanghua/KYUUBI-1542. Closes #1542 1ca1c08a [yanghua] Move FetchIteratorSuite into kyuubi-common module 1307d886 [yanghua] [KYUUBI #1542] Move FetchIterator into kyuubi-common for multi-engine support Authored-by: yanghua Signed-off-by: Cheng Pan --- .../kyuubi/engine/spark/operation/ExecuteScala.scala | 3 +-- .../kyuubi/engine/spark/operation/ExecuteStatement.scala | 4 ++-- .../kyuubi/engine/spark/operation/GetCatalogs.scala | 3 +-- .../apache/kyuubi/engine/spark/operation/GetColumns.scala | 3 +-- .../kyuubi/engine/spark/operation/GetFunctions.scala | 3 +-- .../apache/kyuubi/engine/spark/operation/GetSchemas.scala | 3 +-- .../kyuubi/engine/spark/operation/GetTableTypes.scala | 3 +-- .../apache/kyuubi/engine/spark/operation/GetTables.scala | 3 +-- .../kyuubi/engine/spark/operation/GetTypeInfo.scala | 3 +-- .../kyuubi/engine/spark/operation/PlanOnlyStatement.scala | 3 +-- .../kyuubi/engine/spark/operation/SparkOperation.scala | 3 +-- .../org/apache/kyuubi/operation}/FetchIterator.scala | 8 ++++---- .../org/apache/kyuubi/operation}/FetchIteratorSuite.scala | 2 +- 13 files changed, 17 insertions(+), 27 deletions(-) rename {externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark => kyuubi-common/src/main/scala/org/apache/kyuubi/operation}/FetchIterator.scala (90%) rename {externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark => kyuubi-common/src/test/scala/org/apache/kyuubi/operation}/FetchIteratorSuite.scala (99%) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala index d14f7390e..04f25c54e 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala @@ -23,9 +23,8 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.types.StructType import org.apache.kyuubi.KyuubiSQLException -import org.apache.kyuubi.engine.spark.ArrayFetchIterator import org.apache.kyuubi.engine.spark.repl.KyuubiSparkILoop -import org.apache.kyuubi.operation.OperationType +import org.apache.kyuubi.operation.{ArrayFetchIterator, OperationType} import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala index f43983382..81f123b6b 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala @@ -27,9 +27,9 @@ import org.apache.spark.sql.types._ import org.apache.kyuubi.{KyuubiSQLException, Logging} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.engine.spark.{ArrayFetchIterator, IterableFetchIterator, KyuubiSparkUtil} +import org.apache.kyuubi.engine.spark.KyuubiSparkUtil import org.apache.kyuubi.engine.spark.events.{EventLoggingService, SparkStatementEvent} -import org.apache.kyuubi.operation.{OperationState, OperationType} +import org.apache.kyuubi.operation.{ArrayFetchIterator, IterableFetchIterator, OperationState, OperationType} import org.apache.kyuubi.operation.OperationState.OperationState import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCatalogs.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCatalogs.scala index cbd3fdcc9..9356db6d7 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCatalogs.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCatalogs.scala @@ -19,9 +19,8 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.IterableFetchIterator import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim -import org.apache.kyuubi.operation.OperationType +import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.TABLE_CAT import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetColumns.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetColumns.scala index 471304c0a..debe5c0b3 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetColumns.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetColumns.scala @@ -19,9 +19,8 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.types._ -import org.apache.kyuubi.engine.spark.IterableFetchIterator import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim -import org.apache.kyuubi.operation.OperationType +import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetFunctions.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetFunctions.scala index f7ff1fb14..d4bce5aa2 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetFunctions.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetFunctions.scala @@ -22,8 +22,7 @@ import java.sql.DatabaseMetaData import org.apache.spark.sql.Row import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.IterableFetchIterator -import org.apache.kyuubi.operation.OperationType +import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetSchemas.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetSchemas.scala index 0ac7abbb3..1eaac5f86 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetSchemas.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetSchemas.scala @@ -19,9 +19,8 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.IterableFetchIterator import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim -import org.apache.kyuubi.operation.OperationType +import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTableTypes.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTableTypes.scala index 6ad3a6cf4..f77cd727a 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTableTypes.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTableTypes.scala @@ -20,9 +20,8 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.Row import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.IterableFetchIterator import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim -import org.apache.kyuubi.operation.OperationType +import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala index fb57367a5..d8f56aed5 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala @@ -19,9 +19,8 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.IterableFetchIterator import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim -import org.apache.kyuubi.operation.OperationType +import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTypeInfo.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTypeInfo.scala index 5d0ad510e..5088bbbae 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTypeInfo.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTypeInfo.scala @@ -22,8 +22,7 @@ import java.sql.Types._ import org.apache.spark.sql.Row import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.IterableFetchIterator -import org.apache.kyuubi.operation.OperationType +import org.apache.kyuubi.operation.{IterableFetchIterator, OperationType} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala index e98c8b28c..95cfdae89 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala @@ -22,8 +22,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.types.StructType import org.apache.kyuubi.config.KyuubiConf.OperationModes.{ANALYZE, OperationMode, OPTIMIZE, PARSE} -import org.apache.kyuubi.engine.spark.{ArrayFetchIterator, IterableFetchIterator} -import org.apache.kyuubi.operation.OperationType +import org.apache.kyuubi.operation.{ArrayFetchIterator, IterableFetchIterator, OperationType} import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala index a6238824f..d680bc205 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala @@ -26,10 +26,9 @@ import org.apache.spark.sql.{DataFrame, Row, SparkSession} import org.apache.spark.sql.types.StructType import org.apache.kyuubi.{KyuubiSQLException, Utils} -import org.apache.kyuubi.engine.spark.FetchIterator import org.apache.kyuubi.engine.spark.operation.SparkOperation.TIMEZONE_KEY import org.apache.kyuubi.engine.spark.session.SparkSessionImpl -import org.apache.kyuubi.operation.{AbstractOperation, OperationState} +import org.apache.kyuubi.operation.{AbstractOperation, FetchIterator, OperationState} import org.apache.kyuubi.operation.FetchOrientation._ import org.apache.kyuubi.operation.OperationState.OperationState import org.apache.kyuubi.operation.OperationType.OperationType diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/FetchIterator.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/FetchIterator.scala similarity index 90% rename from externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/FetchIterator.scala rename to kyuubi-common/src/main/scala/org/apache/kyuubi/operation/FetchIterator.scala index 47a08483d..669c8b868 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/FetchIterator.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/FetchIterator.scala @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.kyuubi.engine.spark +package org.apache.kyuubi.operation /** * Borrowed from Apache Spark, see SPARK-33655 */ -sealed private[engine] trait FetchIterator[A] extends Iterator[A] { +sealed trait FetchIterator[A] extends Iterator[A] { /** * Begin a fetch block, forward from the current position. @@ -50,7 +50,7 @@ sealed private[engine] trait FetchIterator[A] extends Iterator[A] { def getPosition: Long } -private[engine] class ArrayFetchIterator[A](src: Array[A]) extends FetchIterator[A] { +class ArrayFetchIterator[A](src: Array[A]) extends FetchIterator[A] { private var fetchStart: Long = 0 private var position: Long = 0 @@ -74,7 +74,7 @@ private[engine] class ArrayFetchIterator[A](src: Array[A]) extends FetchIterator } } -private[engine] class IterableFetchIterator[A](iterable: Iterable[A]) extends FetchIterator[A] { +class IterableFetchIterator[A](iterable: Iterable[A]) extends FetchIterator[A] { private var iter: Iterator[A] = iterable.iterator private var fetchStart: Long = 0 diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/FetchIteratorSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/FetchIteratorSuite.scala similarity index 99% rename from externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/FetchIteratorSuite.scala rename to kyuubi-common/src/test/scala/org/apache/kyuubi/operation/FetchIteratorSuite.scala index e615898e8..9a6b39b01 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/FetchIteratorSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/FetchIteratorSuite.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kyuubi.engine.spark +package org.apache.kyuubi.operation import org.apache.kyuubi.KyuubiFunSuite