[KYUUBI #1565] Move time functions to RowSetUtils and move schema to spark package

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->
1. move time functions from Rowset to RowSetUtils
2. move schema to spark package

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [X] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #1566 from hddong/move-time-functions.

Closes #1565

75de2e8d [hongdongdong] private
8b16217b [hongdongdong] [KYUUBI #1565] Move time functions to RowSetUtils and move schema to spark package

Authored-by: hongdongdong <hongdongdong@cmss.chinamobile.com>
Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
hongdongdong 2021-12-15 21:31:06 +08:00 committed by Cheng Pan
parent 6c235e5925
commit 11e208a688
No known key found for this signature in database
GPG Key ID: 8001952629BCC75D
7 changed files with 42 additions and 43 deletions

View File

@ -27,13 +27,14 @@ import org.apache.spark.sql.types.StructType
import org.apache.kyuubi.{KyuubiSQLException, Utils}
import org.apache.kyuubi.engine.spark.operation.SparkOperation.TIMEZONE_KEY
import org.apache.kyuubi.engine.spark.schema.RowSet
import org.apache.kyuubi.engine.spark.schema.SchemaHelper
import org.apache.kyuubi.engine.spark.session.SparkSessionImpl
import org.apache.kyuubi.operation.{AbstractOperation, FetchIterator, OperationState}
import org.apache.kyuubi.operation.FetchOrientation._
import org.apache.kyuubi.operation.OperationState.OperationState
import org.apache.kyuubi.operation.OperationType.OperationType
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.schema.{RowSet, SchemaHelper}
import org.apache.kyuubi.session.Session
abstract class SparkOperation(opType: OperationType, session: Session)

View File

@ -15,25 +15,22 @@
* limitations under the License.
*/
package org.apache.kyuubi.schema
package org.apache.kyuubi.engine.spark.schema
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.sql.Timestamp
import java.text.SimpleDateFormat
import java.time.{Instant, LocalDate, ZoneId}
import java.time.chrono.IsoChronology
import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder}
import java.time.temporal.ChronoField
import java.util.{Date, Locale}
import java.util.Date
import scala.collection.JavaConverters._
import scala.language.implicitConversions
import org.apache.hive.service.rpc.thrift._
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
import org.apache.kyuubi.util.RowSetUtils._
object RowSet {
def toTRowSet(
@ -146,10 +143,6 @@ object RowSet {
ret
}
implicit private def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = {
ByteBuffer.wrap(bitSet.toByteArray)
}
private def toTColumnValue(
ordinal: Int,
row: Row,
@ -206,29 +199,6 @@ object RowSet {
}
}
private def createBuilder(): DateTimeFormatterBuilder = {
new DateTimeFormatterBuilder().parseCaseInsensitive()
}
private lazy val dateFormatter = {
createBuilder().appendPattern("yyyy-MM-dd")
.toFormatter(Locale.US)
.withChronology(IsoChronology.INSTANCE)
}
private lazy val simpleDateFormatter = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
private lazy val timestampFormatter: DateTimeFormatter = {
createBuilder().appendPattern("yyyy-MM-dd HH:mm:ss")
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
.toFormatter(Locale.US)
.withChronology(IsoChronology.INSTANCE)
}
private lazy val simpleTimestampFormatter = {
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
}
/**
* A simpler impl of Spark's toHiveString
*/

View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.kyuubi.schema
package org.apache.kyuubi.engine.spark.schema
import java.util.Collections

View File

@ -23,7 +23,7 @@ import org.apache.spark.sql.types.StructField
import org.apache.kyuubi.Logging
import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.sparkMajorMinorVersion
import org.apache.kyuubi.schema.SchemaHelper
import org.apache.kyuubi.engine.spark.schema.SchemaHelper
/**
* A shim that defines the interface interact with Spark's catalogs

View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.kyuubi.schema
package org.apache.kyuubi.engine.spark.schema
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
@ -30,7 +30,7 @@ import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.CalendarInterval
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.schema.RowSet.toHiveString
import org.apache.kyuubi.engine.spark.schema.RowSet.toHiveString
class RowSetSuite extends KyuubiFunSuite {

View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.kyuubi.schema
package org.apache.kyuubi.engine.spark.schema
import scala.collection.JavaConverters._
@ -23,11 +23,10 @@ import org.apache.hive.service.rpc.thrift.{TCLIServiceConstants, TTypeId}
import org.apache.spark.sql.types._
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.engine.spark.schema.SchemaHelper._
class SchemaHelperSuite extends KyuubiFunSuite {
import SchemaHelper._
val innerSchema: StructType = new StructType()
.add("a", StringType, nullable = true, "")
.add("b", IntegerType, nullable = true, "")

View File

@ -18,10 +18,39 @@
package org.apache.kyuubi.util
import java.nio.ByteBuffer
import java.text.SimpleDateFormat
import java.time.chrono.IsoChronology
import java.time.format.DateTimeFormatter
import java.time.format.DateTimeFormatterBuilder
import java.time.temporal.ChronoField
import java.util.Locale
import scala.language.implicitConversions
object RowSetUtils {
private[kyuubi] object RowSetUtils {
lazy val dateFormatter = {
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd")
.toFormatter(Locale.US)
.withChronology(IsoChronology.INSTANCE)
}
lazy val simpleDateFormatter = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
lazy val timestampFormatter: DateTimeFormatter = {
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss")
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
.toFormatter(Locale.US)
.withChronology(IsoChronology.INSTANCE)
}
lazy val simpleTimestampFormatter = {
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
}
private def createDateTimeFormatterBuilder(): DateTimeFormatterBuilder = {
new DateTimeFormatterBuilder().parseCaseInsensitive()
}
implicit def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = {
ByteBuffer.wrap(bitSet.toByteArray)