From 6e5f87d6b4e421c7089875ecc957c9dac0356002 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Fri, 17 Feb 2023 10:12:09 +0800 Subject: [PATCH] [KYUUBI #4344] Expose exec pool work queue size metrics ### _Why are the changes needed?_ It can help to know the backend pressure if the exec pool is full. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request Closes #4344 from turboFei/wait_queue. Closes #4344 161e3808a [fwang12] nit 6d122e238 [fwang12] save 55a4b499d [fwang12] version 668ff8bfe [fwang12] save 9f56b98a8 [fwang12] save a401771ec [fwang12] wait Authored-by: fwang12 Signed-off-by: fwang12 --- docs/monitor/metrics.md | 1 + .../scala/org/apache/spark/ui/EnginePage.scala | 4 ++++ .../apache/kyuubi/session/SessionManager.scala | 5 +++++ .../kyuubi/metrics/MetricsConstants.scala | 1 + .../client/api/v1/dto/ExecPoolStatistic.java | 17 ++++++++++++++--- .../kyuubi/server/api/v1/SessionsResource.scala | 3 ++- .../kyuubi/session/KyuubiSessionManager.scala | 1 + 7 files changed, 28 insertions(+), 4 deletions(-) diff --git a/docs/monitor/metrics.md b/docs/monitor/metrics.md index 1d1fa326a..f128fd1a4 100644 --- a/docs/monitor/metrics.md +++ b/docs/monitor/metrics.md @@ -44,6 +44,7 @@ These metrics include: |--------------------------------------------------|----------------------------------------|-----------|-------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `kyuubi.exec.pool.threads.alive` | | gauge | 1.2.0 |
threads keepAlive in the backend executive thread pool
| | `kyuubi.exec.pool.threads.active` | | gauge | 1.2.0 |
threads active in the backend executive thread pool
| +| `kyuubi.exec.pool.work_queue.size` | | gauge | 1.7.0 |
work queue size in the backend executive thread pool
| | `kyuubi.connection.total` | | counter | 1.2.0 |
cumulative connection count
| | `kyuubi.connection.total` | `${sessionType}` | counter | 1.7.0 |
cumulative connection count with session type `${sessionType}`
| | `kyuubi.connection.opened` | | gauge | 1.2.0 |
current active connection count
| diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala index 0aba0c7c5..a2a2931f4 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala @@ -84,6 +84,10 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") { Background execution pool threads active: {engine.backendService.sessionManager.getActiveCount} +
  • + Background execution pool work queue size: + {engine.backendService.sessionManager.getWorkQueueSize} +
    • }.getOrElse(Seq.empty) } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala index 662ac3e58..f8e77dd63 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala @@ -172,6 +172,11 @@ abstract class SessionManager(name: String) extends CompositeService(name) { execPool.getActiveCount } + def getWorkQueueSize: Int = { + assert(execPool != null) + execPool.getQueue.size() + } + private var _confRestrictList: Set[String] = _ private var _confIgnoreList: Set[String] = _ private var _batchConfIgnoreList: Set[String] = _ diff --git a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConstants.scala b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConstants.scala index 62c67266f..e97fd28ea 100644 --- a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConstants.scala +++ b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConstants.scala @@ -29,6 +29,7 @@ object MetricsConstants { final val EXEC_POOL_ALIVE: String = KYUUBI + "exec.pool.threads.alive" final val EXEC_POOL_ACTIVE: String = KYUUBI + "exec.pool.threads.active" + final val EXEC_POOL_WORK_QUEUE_SIZE: String = KYUUBI + "exec.pool.work_queue.size" final private val CONN = KYUUBI + "connection." final private val THRIFT_HTTP_CONN = KYUUBI + "thrift.http.connection." diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ExecPoolStatistic.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ExecPoolStatistic.java index ee8a9f007..a40811f92 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ExecPoolStatistic.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ExecPoolStatistic.java @@ -24,12 +24,14 @@ import org.apache.commons.lang3.builder.ToStringStyle; public class ExecPoolStatistic { private int execPoolSize; private int execPoolActiveCount; + private int execPoolWorkQueueSize; public ExecPoolStatistic() {} - public ExecPoolStatistic(int execPoolSize, int execPoolActiveCount) { + public ExecPoolStatistic(int execPoolSize, int execPoolActiveCount, int execPoolWorkQueueSize) { this.execPoolSize = execPoolSize; this.execPoolActiveCount = execPoolActiveCount; + this.execPoolWorkQueueSize = execPoolWorkQueueSize; } public int getExecPoolSize() { @@ -48,18 +50,27 @@ public class ExecPoolStatistic { this.execPoolActiveCount = execPoolActiveCount; } + public int getExecPoolWorkQueueSize() { + return execPoolWorkQueueSize; + } + + public void setExecPoolWorkQueueSize(int execPoolWorkQueueSize) { + this.execPoolWorkQueueSize = execPoolWorkQueueSize; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ExecPoolStatistic that = (ExecPoolStatistic) o; return getExecPoolSize() == that.getExecPoolSize() - && getExecPoolActiveCount() == that.getExecPoolActiveCount(); + && getExecPoolActiveCount() == that.getExecPoolActiveCount() + && getExecPoolWorkQueueSize() == that.getExecPoolWorkQueueSize(); } @Override public int hashCode() { - return Objects.hash(getExecPoolSize(), getExecPoolActiveCount()); + return Objects.hash(getExecPoolSize(), getExecPoolActiveCount(), getExecPoolWorkQueueSize()); } @Override diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala index dd4a8c3a7..84b19eb00 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala @@ -131,7 +131,8 @@ private[v1] class SessionsResource extends ApiRequestContext with Logging { def execPoolStatistic(): ExecPoolStatistic = { new ExecPoolStatistic( sessionManager.getExecPoolSize, - sessionManager.getActiveCount) + sessionManager.getActiveCount, + sessionManager.getWorkQueueSize) } @ApiResponse( diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala index 54d5b8b24..207ae4c4d 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala @@ -239,6 +239,7 @@ class KyuubiSessionManager private (name: String) extends SessionManager(name) { ms.registerGauge(CONN_OPEN, getOpenSessionCount, 0) ms.registerGauge(EXEC_POOL_ALIVE, getExecPoolSize, 0) ms.registerGauge(EXEC_POOL_ACTIVE, getActiveCount, 0) + ms.registerGauge(EXEC_POOL_WORK_QUEUE_SIZE, getWorkQueueSize, 0) } super.start() }