[KYUUBI #1565] Move time functions to RowSetUtils and move schema to spark package
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> 1. move time functions from Rowset to RowSetUtils 2. move schema to spark package ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [X] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1566 from hddong/move-time-functions. Closes #1565 75de2e8d [hongdongdong] private 8b16217b [hongdongdong] [KYUUBI #1565] Move time functions to RowSetUtils and move schema to spark package Authored-by: hongdongdong <hongdongdong@cmss.chinamobile.com> Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
parent
6c235e5925
commit
11e208a688
@ -27,13 +27,14 @@ import org.apache.spark.sql.types.StructType
|
||||
|
||||
import org.apache.kyuubi.{KyuubiSQLException, Utils}
|
||||
import org.apache.kyuubi.engine.spark.operation.SparkOperation.TIMEZONE_KEY
|
||||
import org.apache.kyuubi.engine.spark.schema.RowSet
|
||||
import org.apache.kyuubi.engine.spark.schema.SchemaHelper
|
||||
import org.apache.kyuubi.engine.spark.session.SparkSessionImpl
|
||||
import org.apache.kyuubi.operation.{AbstractOperation, FetchIterator, OperationState}
|
||||
import org.apache.kyuubi.operation.FetchOrientation._
|
||||
import org.apache.kyuubi.operation.OperationState.OperationState
|
||||
import org.apache.kyuubi.operation.OperationType.OperationType
|
||||
import org.apache.kyuubi.operation.log.OperationLog
|
||||
import org.apache.kyuubi.schema.{RowSet, SchemaHelper}
|
||||
import org.apache.kyuubi.session.Session
|
||||
|
||||
abstract class SparkOperation(opType: OperationType, session: Session)
|
||||
|
||||
@ -15,25 +15,22 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kyuubi.schema
|
||||
package org.apache.kyuubi.engine.spark.schema
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.sql.Timestamp
|
||||
import java.text.SimpleDateFormat
|
||||
import java.time.{Instant, LocalDate, ZoneId}
|
||||
import java.time.chrono.IsoChronology
|
||||
import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder}
|
||||
import java.time.temporal.ChronoField
|
||||
import java.util.{Date, Locale}
|
||||
import java.util.Date
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.language.implicitConversions
|
||||
|
||||
import org.apache.hive.service.rpc.thrift._
|
||||
import org.apache.spark.sql.Row
|
||||
import org.apache.spark.sql.types._
|
||||
|
||||
import org.apache.kyuubi.util.RowSetUtils._
|
||||
|
||||
object RowSet {
|
||||
|
||||
def toTRowSet(
|
||||
@ -146,10 +143,6 @@ object RowSet {
|
||||
ret
|
||||
}
|
||||
|
||||
implicit private def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = {
|
||||
ByteBuffer.wrap(bitSet.toByteArray)
|
||||
}
|
||||
|
||||
private def toTColumnValue(
|
||||
ordinal: Int,
|
||||
row: Row,
|
||||
@ -206,29 +199,6 @@ object RowSet {
|
||||
}
|
||||
}
|
||||
|
||||
private def createBuilder(): DateTimeFormatterBuilder = {
|
||||
new DateTimeFormatterBuilder().parseCaseInsensitive()
|
||||
}
|
||||
|
||||
private lazy val dateFormatter = {
|
||||
createBuilder().appendPattern("yyyy-MM-dd")
|
||||
.toFormatter(Locale.US)
|
||||
.withChronology(IsoChronology.INSTANCE)
|
||||
}
|
||||
|
||||
private lazy val simpleDateFormatter = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
|
||||
|
||||
private lazy val timestampFormatter: DateTimeFormatter = {
|
||||
createBuilder().appendPattern("yyyy-MM-dd HH:mm:ss")
|
||||
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
|
||||
.toFormatter(Locale.US)
|
||||
.withChronology(IsoChronology.INSTANCE)
|
||||
}
|
||||
|
||||
private lazy val simpleTimestampFormatter = {
|
||||
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
|
||||
}
|
||||
|
||||
/**
|
||||
* A simpler impl of Spark's toHiveString
|
||||
*/
|
||||
@ -15,7 +15,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kyuubi.schema
|
||||
package org.apache.kyuubi.engine.spark.schema
|
||||
|
||||
import java.util.Collections
|
||||
|
||||
@ -23,7 +23,7 @@ import org.apache.spark.sql.types.StructField
|
||||
|
||||
import org.apache.kyuubi.Logging
|
||||
import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.sparkMajorMinorVersion
|
||||
import org.apache.kyuubi.schema.SchemaHelper
|
||||
import org.apache.kyuubi.engine.spark.schema.SchemaHelper
|
||||
|
||||
/**
|
||||
* A shim that defines the interface interact with Spark's catalogs
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kyuubi.schema
|
||||
package org.apache.kyuubi.engine.spark.schema
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.charset.StandardCharsets
|
||||
@ -30,7 +30,7 @@ import org.apache.spark.sql.types._
|
||||
import org.apache.spark.unsafe.types.CalendarInterval
|
||||
|
||||
import org.apache.kyuubi.KyuubiFunSuite
|
||||
import org.apache.kyuubi.schema.RowSet.toHiveString
|
||||
import org.apache.kyuubi.engine.spark.schema.RowSet.toHiveString
|
||||
|
||||
class RowSetSuite extends KyuubiFunSuite {
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.kyuubi.schema
|
||||
package org.apache.kyuubi.engine.spark.schema
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
@ -23,11 +23,10 @@ import org.apache.hive.service.rpc.thrift.{TCLIServiceConstants, TTypeId}
|
||||
import org.apache.spark.sql.types._
|
||||
|
||||
import org.apache.kyuubi.KyuubiFunSuite
|
||||
import org.apache.kyuubi.engine.spark.schema.SchemaHelper._
|
||||
|
||||
class SchemaHelperSuite extends KyuubiFunSuite {
|
||||
|
||||
import SchemaHelper._
|
||||
|
||||
val innerSchema: StructType = new StructType()
|
||||
.add("a", StringType, nullable = true, "")
|
||||
.add("b", IntegerType, nullable = true, "")
|
||||
@ -18,10 +18,39 @@
|
||||
package org.apache.kyuubi.util
|
||||
|
||||
import java.nio.ByteBuffer
|
||||
import java.text.SimpleDateFormat
|
||||
import java.time.chrono.IsoChronology
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.time.format.DateTimeFormatterBuilder
|
||||
import java.time.temporal.ChronoField
|
||||
import java.util.Locale
|
||||
|
||||
import scala.language.implicitConversions
|
||||
|
||||
object RowSetUtils {
|
||||
private[kyuubi] object RowSetUtils {
|
||||
|
||||
lazy val dateFormatter = {
|
||||
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd")
|
||||
.toFormatter(Locale.US)
|
||||
.withChronology(IsoChronology.INSTANCE)
|
||||
}
|
||||
|
||||
lazy val simpleDateFormatter = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
|
||||
|
||||
lazy val timestampFormatter: DateTimeFormatter = {
|
||||
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss")
|
||||
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
|
||||
.toFormatter(Locale.US)
|
||||
.withChronology(IsoChronology.INSTANCE)
|
||||
}
|
||||
|
||||
lazy val simpleTimestampFormatter = {
|
||||
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
|
||||
}
|
||||
|
||||
private def createDateTimeFormatterBuilder(): DateTimeFormatterBuilder = {
|
||||
new DateTimeFormatterBuilder().parseCaseInsensitive()
|
||||
}
|
||||
|
||||
implicit def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = {
|
||||
ByteBuffer.wrap(bitSet.toByteArray)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user