[KYUUBI #4344] Expose exec pool work queue size metrics

### _Why are the changes needed?_

It can help to know the backend pressure if the exec pool is full.

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #4344 from turboFei/wait_queue.

Closes #4344

161e3808a [fwang12] nit
6d122e238 [fwang12] save
55a4b499d [fwang12] version
668ff8bfe [fwang12] save
9f56b98a8 [fwang12] save
a401771ec [fwang12] wait

Authored-by: fwang12 <fwang12@ebay.com>
Signed-off-by: fwang12 <fwang12@ebay.com>

docs/monitor/metrics.md

@@ -44,6 +44,7 @@ These metrics include:
 |--------------------------------------------------|----------------------------------------|-----------|-------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `kyuubi.exec.pool.threads.alive`                 |                                        | gauge     | 1.2.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> threads keepAlive in the backend executive thread pool</div>                                                                                                                                                          |
 | `kyuubi.exec.pool.threads.active`                |                                        | gauge     | 1.2.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> threads active in the backend executive thread pool</div>                                                                                                                                                             |
+| `kyuubi.exec.pool.work_queue.size`               |                                        | gauge     | 1.7.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> work queue size in the backend executive thread pool</div>                                                                                                                                                            |
 | `kyuubi.connection.total`                        |                                        | counter   | 1.2.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'>  cumulative connection count</div>                                                                                                                                                                                    |
 | `kyuubi.connection.total`                        | `${sessionType}`                       | counter   | 1.7.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> cumulative connection count with session type `${sessionType}`</div>                                                                                                                                                  |
 | `kyuubi.connection.opened`                       |                                        | gauge     | 1.2.0 | <div style='width: 150pt;word-wrap: break-word;white-space: normal'> current active connection count</div>                                                                                                                                                                                 |

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala

@@ -84,6 +84,10 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") {
           <strong>Background execution pool threads active: </strong>
           {engine.backendService.sessionManager.getActiveCount}
         </li>
+          <li>
+            <strong>Background execution pool work queue size: </strong>
+            {engine.backendService.sessionManager.getWorkQueueSize}
+          </li>
       }.getOrElse(Seq.empty)
     }
     </ul>

kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala

@@ -172,6 +172,11 @@ abstract class SessionManager(name: String) extends CompositeService(name) {
     execPool.getActiveCount
   }
 
+  def getWorkQueueSize: Int = {
+    assert(execPool != null)
+    execPool.getQueue.size()
+  }
+
   private var _confRestrictList: Set[String] = _
   private var _confIgnoreList: Set[String] = _
   private var _batchConfIgnoreList: Set[String] = _

kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConstants.scala

@@ -29,6 +29,7 @@ object MetricsConstants {
 
   final val EXEC_POOL_ALIVE: String = KYUUBI + "exec.pool.threads.alive"
   final val EXEC_POOL_ACTIVE: String = KYUUBI + "exec.pool.threads.active"
+  final val EXEC_POOL_WORK_QUEUE_SIZE: String = KYUUBI + "exec.pool.work_queue.size"
 
   final private val CONN = KYUUBI + "connection."
   final private val THRIFT_HTTP_CONN = KYUUBI + "thrift.http.connection."

kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ExecPoolStatistic.java

@@ -24,12 +24,14 @@ import org.apache.commons.lang3.builder.ToStringStyle;
 public class ExecPoolStatistic {
   private int execPoolSize;
   private int execPoolActiveCount;
+  private int execPoolWorkQueueSize;
 
   public ExecPoolStatistic() {}
 
-  public ExecPoolStatistic(int execPoolSize, int execPoolActiveCount) {
+  public ExecPoolStatistic(int execPoolSize, int execPoolActiveCount, int execPoolWorkQueueSize) {
     this.execPoolSize = execPoolSize;
     this.execPoolActiveCount = execPoolActiveCount;
+    this.execPoolWorkQueueSize = execPoolWorkQueueSize;
   }
 
   public int getExecPoolSize() {
@@ -48,18 +50,27 @@ public class ExecPoolStatistic {
     this.execPoolActiveCount = execPoolActiveCount;
   }
 
+  public int getExecPoolWorkQueueSize() {
+    return execPoolWorkQueueSize;
+  }
+
+  public void setExecPoolWorkQueueSize(int execPoolWorkQueueSize) {
+    this.execPoolWorkQueueSize = execPoolWorkQueueSize;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
     if (o == null || getClass() != o.getClass()) return false;
     ExecPoolStatistic that = (ExecPoolStatistic) o;
     return getExecPoolSize() == that.getExecPoolSize()
-        && getExecPoolActiveCount() == that.getExecPoolActiveCount();
+        && getExecPoolActiveCount() == that.getExecPoolActiveCount()
+        && getExecPoolWorkQueueSize() == that.getExecPoolWorkQueueSize();
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(getExecPoolSize(), getExecPoolActiveCount());
+    return Objects.hash(getExecPoolSize(), getExecPoolActiveCount(), getExecPoolWorkQueueSize());
   }
 
   @Override

kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/SessionsResource.scala

@@ -131,7 +131,8 @@ private[v1] class SessionsResource extends ApiRequestContext with Logging {
   def execPoolStatistic(): ExecPoolStatistic = {
     new ExecPoolStatistic(
       sessionManager.getExecPoolSize,
-      sessionManager.getActiveCount)
+      sessionManager.getActiveCount,
+      sessionManager.getWorkQueueSize)
   }
 
   @ApiResponse(

kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala

@@ -239,6 +239,7 @@ class KyuubiSessionManager private (name: String) extends SessionManager(name) {
       ms.registerGauge(CONN_OPEN, getOpenSessionCount, 0)
       ms.registerGauge(EXEC_POOL_ALIVE, getExecPoolSize, 0)
       ms.registerGauge(EXEC_POOL_ACTIVE, getActiveCount, 0)
+      ms.registerGauge(EXEC_POOL_WORK_QUEUE_SIZE, getWorkQueueSize, 0)
     }
     super.start()
   }