From 250d2f17b67e4b694c29cda9a03e3b4af50d014d Mon Sep 17 00:00:00 2001 From: Fu Chen Date: Mon, 17 Jan 2022 18:40:32 +0800 Subject: [PATCH] [KYUUBI #1776] Improve EnginePage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### _Why are the changes needed?_ Improve the UI page of `Kyuubi Query Engine`. Currently, the column named `Query Execution` in tab `SQL Statistics` is not readable 1. Replace column `Query Execution` with `Query Details` and point it to the Spark SQL detail page 2. Add a new column `Failure Reason` to display an exception message when the query fails. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [x] Add screenshots for manual tests if appropriate ![截屏2022-01-17 下午2 53 43](https://user-images.githubusercontent.com/8537877/149723476-7dd1aaab-b141-4ca3-a824-cbd393378dd8.png) - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1776 from cfmcgrady/ui. Closes #1776 efab75a7 [Fu Chen] trigger GitHub actions 5b7001b2 [Fu Chen] fix ut d9ccff89 [Fu Chen] fix style 717b916d [Fu Chen] Improve EnginePage Authored-by: Fu Chen Signed-off-by: ulysses-you --- .../spark/events/SparkOperationEvent.scala | 12 +++++++----- .../spark/operation/ExecuteStatement.scala | 3 ++- .../spark/kyuubi/SQLOperationListener.scala | 2 ++ .../scala/org/apache/spark/ui/EnginePage.scala | 17 ++++++++++++++--- .../spark/events/EngineEventsStoreSuite.scala | 16 ++++++++-------- .../org/apache/spark/ui/EngineTabSuite.scala | 2 +- 6 files changed, 34 insertions(+), 18 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SparkOperationEvent.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SparkOperationEvent.scala index 4b3a598da..87fbc87c7 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SparkOperationEvent.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SparkOperationEvent.scala @@ -17,7 +17,7 @@ package org.apache.kyuubi.engine.spark.events -import org.apache.spark.sql.{DataFrame, Encoders} +import org.apache.spark.sql.Encoders import org.apache.spark.sql.types.StructType import org.apache.kyuubi.Utils @@ -42,7 +42,7 @@ import org.apache.kyuubi.engine.spark.operation.SparkOperation * @param exception: caught exception if have * @param sessionId the identifier of the parent session * @param sessionUser the authenticated client user - * @param queryExecution the query execution of this operation + * @param executionId the query execution id of this operation */ case class SparkOperationEvent( statementId: String, @@ -56,7 +56,7 @@ case class SparkOperationEvent( exception: Option[Throwable], sessionId: String, sessionUser: String, - queryExecution: String) extends KyuubiSparkEvent { + executionId: Option[Long]) extends KyuubiSparkEvent { override def schema: StructType = Encoders.product[SparkOperationEvent].schema override def partitions: Seq[(String, String)] = @@ -72,7 +72,9 @@ case class SparkOperationEvent( } object SparkOperationEvent { - def apply(operation: SparkOperation, result: Option[DataFrame] = None): SparkOperationEvent = { + def apply( + operation: SparkOperation, + executionId: Option[Long] = None): SparkOperationEvent = { val session = operation.getSession val status = operation.getStatus new SparkOperationEvent( @@ -87,6 +89,6 @@ object SparkOperationEvent { status.exception, session.handle.identifier.toString, session.user, - result.map(_.queryExecution.toString).orNull) + executionId) } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala index 495ae48f3..450ec4bc0 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala @@ -146,6 +146,7 @@ class ExecuteStatement( override def setState(newState: OperationState): Unit = { super.setState(newState) - EventLoggingService.onEvent(SparkOperationEvent(this, Option(result))) + EventLoggingService.onEvent( + SparkOperationEvent(this, operationListener.getExecutionId)) } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala index 0337b34f6..7950db9b3 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala @@ -44,6 +44,8 @@ class SQLOperationListener( private val activeStages = new java.util.HashSet[Int]() private var executionId: Option[Long] = None + def getExecutionId: Option[Long] = executionId + // For broadcast, Spark will introduce a new runId as SPARK_JOB_GROUP_ID, see: // https://github.com/apache/spark/pull/24595, So we will miss these logs. // TODO: Fix this until the below ticket resolved diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala index f9c4c5667..b18888f3a 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala @@ -327,7 +327,8 @@ private class StatementStatsPagedTable( ("Duration", true, None), ("Statement", true, None), ("State", true, None), - ("Query Execution", true, None)) + ("Query Details", true, None), + ("Failure Reason", true, None)) headerStatRow( sqlTableHeadersAndTooltips, @@ -365,8 +366,19 @@ private class StatementStatsPagedTable( {event.state} - {event.queryExecution} + { + if (event.executionId.isDefined) { + + {event.executionId.get} + + } + } + {if (event.exception.isDefined) errorMessageCell(event.exception.get.getMessage)} } @@ -447,7 +459,6 @@ private class StatementStatsTableDataSource( case "Duration" => Ordering.by(_.duration) case "Statement" => Ordering.by(_.statement) case "State" => Ordering.by(_.state) - case "Query Execution" => Ordering.by(_.queryExecution) case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn") } if (desc) { diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/events/EngineEventsStoreSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/events/EngineEventsStoreSuite.scala index d84a9ef73..881ff0ed7 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/events/EngineEventsStoreSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/events/EngineEventsStoreSuite.scala @@ -94,7 +94,7 @@ class EngineEventsStoreSuite extends KyuubiFunSuite { None, "sid1", "a", - "") + None) val s2 = SparkOperationEvent( "ea2", "select 2", @@ -107,7 +107,7 @@ class EngineEventsStoreSuite extends KyuubiFunSuite { None, "sid1", "c", - "") + None) val s3 = SparkOperationEvent( "ea3", "select 3", @@ -120,7 +120,7 @@ class EngineEventsStoreSuite extends KyuubiFunSuite { None, "sid1", "b", - "") + None) store.saveStatement(s1) store.saveStatement(s2) @@ -149,7 +149,7 @@ class EngineEventsStoreSuite extends KyuubiFunSuite { None, "sid1", "a", - "") + None) store.saveStatement(s) } @@ -174,7 +174,7 @@ class EngineEventsStoreSuite extends KyuubiFunSuite { None, "sid1", "a", - "")) + None)) store.saveStatement(SparkOperationEvent( "s2", "select 1", @@ -187,7 +187,7 @@ class EngineEventsStoreSuite extends KyuubiFunSuite { None, "sid1", "a", - "")) + None)) store.saveStatement(SparkOperationEvent( "s3", "select 1", @@ -200,7 +200,7 @@ class EngineEventsStoreSuite extends KyuubiFunSuite { None, "sid1", "a", - "")) + None)) store.saveStatement(SparkOperationEvent( "s4", "select 1", @@ -213,7 +213,7 @@ class EngineEventsStoreSuite extends KyuubiFunSuite { None, "sid1", "a", - "")) + None)) assert(store.getStatementList.size == 3) assert(store.getStatementList(2).statementId == "s4") diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala index 50ddf0a5c..ca90d92cf 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala @@ -133,7 +133,7 @@ class EngineTabSuite extends WithSparkSQLEngine with HiveJDBCTestHelper { assert(resp.contains("sqlstat")) // check sql stats table title - assert(resp.contains("Query Execution")) + assert(resp.contains("Query Details")) } }