From 73eeea5f5cf46aa9afb75b2ffae36c41d7368ded Mon Sep 17 00:00:00 2001 From: zhang1002 Date: Tue, 23 Nov 2021 17:07:05 +0800 Subject: [PATCH] [KYUUBI #1436] unified the field name as kyuubiStatementEvent ### _Why are the changes needed?_ unified the field name as kyuubiStatementEvent ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1436 from zhang1002/unified-field-name. Closes #1436 4c83e44a [zhang1002] unified the field name as kyuubiStatementEvent Authored-by: zhang1002 Signed-off-by: Kent Yao --- .../kyuubi/engine/spark/events/EngineEventsStore.scala | 2 +- .../engine/spark/events/SparkStatementEvent.scala | 10 +++++----- .../engine/spark/operation/ExecuteStatement.scala | 4 ++-- .../scala/org/apache/spark/kyuubi/ui/EnginePage.scala | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/EngineEventsStore.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/EngineEventsStore.scala index 833e67818..260976d8a 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/EngineEventsStore.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/EngineEventsStore.scala @@ -122,7 +122,7 @@ class EngineEventsStore(conf: KyuubiConf) { val reverseSeq = statements.values().asScala.toSeq.sortBy(_.createTime).reverse // remove finished statements first. - for (event <- reverseSeq if event.endTime != -1L && countToDelete > 0) { + for (event <- reverseSeq if event.completeTime != -1L && countToDelete > 0) { statements.remove(event.statementId) countToDelete -= 1 } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SparkStatementEvent.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SparkStatementEvent.scala index e055eff0e..4995b6983 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SparkStatementEvent.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SparkStatementEvent.scala @@ -29,7 +29,7 @@ import org.apache.kyuubi.Utils * @param sessionId: the identifier of a session * @param createTime: the create time of this statement * @param state: store each state that the sql has - * @param stateTime: the time that the sql's state change + * @param eventTime: the time that the sql's state change * @param queryExecution: contains logicPlan and physicalPlan * @param exception: caught exception if have */ @@ -41,8 +41,8 @@ case class SparkStatementEvent( sessionId: String, createTime: Long, var state: String, - var stateTime: Long, - var endTime: Long = -1L, + var eventTime: Long, + var completeTime: Long = -1L, var queryExecution: String = "", var exception: String = "") extends KyuubiSparkEvent { @@ -51,10 +51,10 @@ case class SparkStatementEvent( ("day", Utils.getDateFromTimestamp(createTime)) :: Nil def duration: Long = { - if (endTime == -1L) { + if (completeTime == -1L) { System.currentTimeMillis - createTime } else { - endTime - createTime + completeTime - createTime } } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala index 5aebcca2a..7022217f8 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala @@ -189,9 +189,9 @@ class ExecuteStatement( override def setState(newState: OperationState): Unit = { super.setState(newState) statementEvent.state = newState.toString - statementEvent.stateTime = lastAccessTime + statementEvent.eventTime = lastAccessTime if (newState == OperationState.ERROR || newState == OperationState.FINISHED) { - statementEvent.endTime = System.currentTimeMillis() + statementEvent.completeTime = System.currentTimeMillis() } EventLoggingService.onEvent(statementEvent) } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EnginePage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EnginePage.scala index a357c0e45..7f870ec63 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EnginePage.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EnginePage.scala @@ -347,7 +347,7 @@ private class StatementStatsPagedTable( {formatDate(sparkStatementEvent.createTime)} - {if (sparkStatementEvent.endTime > 0) formatDate(sparkStatementEvent.endTime)} + {if (sparkStatementEvent.completeTime > 0) formatDate(sparkStatementEvent.completeTime)} {formatDurationVerbose(sparkStatementEvent.duration)} @@ -436,7 +436,7 @@ private class StatementStatsTableDataSource( case "User" => Ordering.by(_.username) case "Statement ID" => Ordering.by(_.statementId) case "Create Time" => Ordering by (_.createTime) - case "Finish Time" => Ordering.by(_.endTime) + case "Finish Time" => Ordering.by(_.completeTime) case "Duration" => Ordering.by(_.duration) case "Statement" => Ordering.by(_.statement) case "State" => Ordering.by(_.state)