From 11e208a68848fc9f009c95c37f92832f10e80a1a Mon Sep 17 00:00:00 2001 From: hongdongdong Date: Wed, 15 Dec 2021 21:31:06 +0800 Subject: [PATCH] [KYUUBI #1565] Move time functions to RowSetUtils and move schema to spark package ### _Why are the changes needed?_ 1. move time functions from Rowset to RowSetUtils 2. move schema to spark package ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [X] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1566 from hddong/move-time-functions. Closes #1565 75de2e8d [hongdongdong] private 8b16217b [hongdongdong] [KYUUBI #1565] Move time functions to RowSetUtils and move schema to spark package Authored-by: hongdongdong Signed-off-by: Cheng Pan --- .../spark/operation/SparkOperation.scala | 3 +- .../{ => engine/spark}/schema/RowSet.scala | 38 ++----------------- .../spark}/schema/SchemaHelper.scala | 2 +- .../engine/spark/shim/SparkCatalogShim.scala | 2 +- .../spark}/schema/RowSetSuite.scala | 4 +- .../spark}/schema/SchemaHelperSuite.scala | 5 +-- .../org/apache/kyuubi/util/RowSetUtils.scala | 31 ++++++++++++++- 7 files changed, 42 insertions(+), 43 deletions(-) rename externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/{ => engine/spark}/schema/RowSet.scala (87%) rename externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/{ => engine/spark}/schema/SchemaHelper.scala (99%) rename externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/{ => engine/spark}/schema/RowSetSuite.scala (98%) rename externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/{ => engine/spark}/schema/SchemaHelperSuite.scala (97%) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala index d680bc205..49325808e 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala @@ -27,13 +27,14 @@ import org.apache.spark.sql.types.StructType import org.apache.kyuubi.{KyuubiSQLException, Utils} import org.apache.kyuubi.engine.spark.operation.SparkOperation.TIMEZONE_KEY +import org.apache.kyuubi.engine.spark.schema.RowSet +import org.apache.kyuubi.engine.spark.schema.SchemaHelper import org.apache.kyuubi.engine.spark.session.SparkSessionImpl import org.apache.kyuubi.operation.{AbstractOperation, FetchIterator, OperationState} import org.apache.kyuubi.operation.FetchOrientation._ import org.apache.kyuubi.operation.OperationState.OperationState import org.apache.kyuubi.operation.OperationType.OperationType import org.apache.kyuubi.operation.log.OperationLog -import org.apache.kyuubi.schema.{RowSet, SchemaHelper} import org.apache.kyuubi.session.Session abstract class SparkOperation(opType: OperationType, session: Session) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/schema/RowSet.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala similarity index 87% rename from externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/schema/RowSet.scala rename to externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala index 9a4d22e45..ba6d9c9f4 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/schema/RowSet.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala @@ -15,25 +15,22 @@ * limitations under the License. */ -package org.apache.kyuubi.schema +package org.apache.kyuubi.engine.spark.schema import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import java.sql.Timestamp -import java.text.SimpleDateFormat import java.time.{Instant, LocalDate, ZoneId} -import java.time.chrono.IsoChronology -import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder} -import java.time.temporal.ChronoField -import java.util.{Date, Locale} +import java.util.Date import scala.collection.JavaConverters._ -import scala.language.implicitConversions import org.apache.hive.service.rpc.thrift._ import org.apache.spark.sql.Row import org.apache.spark.sql.types._ +import org.apache.kyuubi.util.RowSetUtils._ + object RowSet { def toTRowSet( @@ -146,10 +143,6 @@ object RowSet { ret } - implicit private def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = { - ByteBuffer.wrap(bitSet.toByteArray) - } - private def toTColumnValue( ordinal: Int, row: Row, @@ -206,29 +199,6 @@ object RowSet { } } - private def createBuilder(): DateTimeFormatterBuilder = { - new DateTimeFormatterBuilder().parseCaseInsensitive() - } - - private lazy val dateFormatter = { - createBuilder().appendPattern("yyyy-MM-dd") - .toFormatter(Locale.US) - .withChronology(IsoChronology.INSTANCE) - } - - private lazy val simpleDateFormatter = new SimpleDateFormat("yyyy-MM-dd", Locale.US) - - private lazy val timestampFormatter: DateTimeFormatter = { - createBuilder().appendPattern("yyyy-MM-dd HH:mm:ss") - .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) - .toFormatter(Locale.US) - .withChronology(IsoChronology.INSTANCE) - } - - private lazy val simpleTimestampFormatter = { - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US) - } - /** * A simpler impl of Spark's toHiveString */ diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/schema/SchemaHelper.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala similarity index 99% rename from externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/schema/SchemaHelper.scala rename to externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala index e617ab570..4133b576a 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/schema/SchemaHelper.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kyuubi.schema +package org.apache.kyuubi.engine.spark.schema import java.util.Collections diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala index c2263f6bf..831b39c74 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.types.StructField import org.apache.kyuubi.Logging import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.sparkMajorMinorVersion -import org.apache.kyuubi.schema.SchemaHelper +import org.apache.kyuubi.engine.spark.schema.SchemaHelper /** * A shim that defines the interface interact with Spark's catalogs diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/schema/RowSetSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala similarity index 98% rename from externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/schema/RowSetSuite.scala rename to externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala index 77e03d560..96d544bf5 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/schema/RowSetSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kyuubi.schema +package org.apache.kyuubi.engine.spark.schema import java.nio.ByteBuffer import java.nio.charset.StandardCharsets @@ -30,7 +30,7 @@ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.CalendarInterval import org.apache.kyuubi.KyuubiFunSuite -import org.apache.kyuubi.schema.RowSet.toHiveString +import org.apache.kyuubi.engine.spark.schema.RowSet.toHiveString class RowSetSuite extends KyuubiFunSuite { diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/schema/SchemaHelperSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelperSuite.scala similarity index 97% rename from externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/schema/SchemaHelperSuite.scala rename to externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelperSuite.scala index 7595c2f85..3e0f61207 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/schema/SchemaHelperSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelperSuite.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kyuubi.schema +package org.apache.kyuubi.engine.spark.schema import scala.collection.JavaConverters._ @@ -23,11 +23,10 @@ import org.apache.hive.service.rpc.thrift.{TCLIServiceConstants, TTypeId} import org.apache.spark.sql.types._ import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.engine.spark.schema.SchemaHelper._ class SchemaHelperSuite extends KyuubiFunSuite { - import SchemaHelper._ - val innerSchema: StructType = new StructType() .add("a", StringType, nullable = true, "") .add("b", IntegerType, nullable = true, "") diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala index 5cb421c2e..3c1cb83b0 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala @@ -18,10 +18,39 @@ package org.apache.kyuubi.util import java.nio.ByteBuffer +import java.text.SimpleDateFormat +import java.time.chrono.IsoChronology +import java.time.format.DateTimeFormatter +import java.time.format.DateTimeFormatterBuilder +import java.time.temporal.ChronoField +import java.util.Locale import scala.language.implicitConversions -object RowSetUtils { +private[kyuubi] object RowSetUtils { + + lazy val dateFormatter = { + createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd") + .toFormatter(Locale.US) + .withChronology(IsoChronology.INSTANCE) + } + + lazy val simpleDateFormatter = new SimpleDateFormat("yyyy-MM-dd", Locale.US) + + lazy val timestampFormatter: DateTimeFormatter = { + createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss") + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .toFormatter(Locale.US) + .withChronology(IsoChronology.INSTANCE) + } + + lazy val simpleTimestampFormatter = { + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US) + } + + private def createDateTimeFormatterBuilder(): DateTimeFormatterBuilder = { + new DateTimeFormatterBuilder().parseCaseInsensitive() + } implicit def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = { ByteBuffer.wrap(bitSet.toByteArray)