[KYUUBI #1205] Support to show more engine submission failure log for issue tracking

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->
Customers might meet exception when launching a spark engine, especially for beginner, now kyuubi only show the last line of engine log and it is difficult to find the root cause in customer side.

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #1205 from turboFei/show_more_engine_log.

Closes #1205

72002dd6 [fwang12] address comments
a74fa1de [fwang12] update docs
fd967371 [fwang12] refactor
434a093d [fwang12] update docs
880c8cc2 [fwang12] show more engine failure log for debug

Authored-by: fwang12 <fwang12@ebay.com>
Signed-off-by: fwang12 <fwang12@ebay.com>

docs/deployment/settings.md

@@ -286,6 +286,7 @@ kyuubi\.session\.engine<br>\.login\.timeout|<div style='width: 65pt;word-wrap: b
 kyuubi\.session\.engine<br>\.share\.level|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>USER</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) - Using kyuubi.engine.share.level instead</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.0.0</div>
 kyuubi\.session\.engine<br>\.spark\.main\.resource|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>&lt;undefined&gt;</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>The package used to create Spark SQL engine remote application. If it is undefined, Kyuubi will use the default</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.0.0</div>
 kyuubi\.session\.engine<br>\.startup\.error\.max<br>\.size|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>8192</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>During engine bootstrapping, if error occurs, using this config to limit the length error message(characters).</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.1.0</div>
+kyuubi\.session\.engine<br>\.startup\.maxLogLines|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>10</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>The maximum number of engine log lines when errors occur during engine startup phase. Note that this max lines is for client-side to help track engine startup issue.</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
 kyuubi\.session\.idle<br>\.timeout|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT6H</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>session idle timeout, it will be closed when it's not accessed for this duration</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.2.0</div>
 kyuubi\.session\.name|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>&lt;undefined&gt;</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>A human readable name of session and we use empty string by default. This name will be recorded in event. Note that, we only apply this value from session conf.</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.4.0</div>
 kyuubi\.session<br>\.timeout|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT6H</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated)session timeout, it will be closed when it's not accessed for this duration</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.0.0</div>

kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

@@ -594,6 +594,15 @@ object KyuubiConf {
       .toSequence()
       .createWithDefault(Nil)
 
+  val SESSION_ENGINE_STARTUP_MAX_LOG_LINES: ConfigEntry[Int] =
+    buildConf("session.engine.startup.maxLogLines")
+      .doc("The maximum number of engine log lines when errors occur during engine startup phase." +
+        " Note that this max lines is for client-side to help track engine startup issue.")
+      .version("1.4.0")
+      .intConf
+      .checkValue(_ > 0, "the maximum must be positive integer.")
+      .createWithDefault(10)
+
   val SERVER_EXEC_POOL_SIZE: ConfigEntry[Int] =
     buildConf("backend.server.exec.pool.size")
       .doc("Number of threads in the operation execution thread pool of Kyuubi server")

kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala

@@ -24,6 +24,7 @@ import java.nio.file.{Files, Path}
 import scala.collection.JavaConverters._
 import scala.util.matching.Regex
 
+import com.google.common.collect.EvictingQueue
 import org.apache.commons.lang3.StringUtils.containsIgnoreCase
 
 import org.apache.kyuubi.{KyuubiSQLException, Logging}
@@ -64,7 +65,9 @@ trait ProcBuilder {
   }
 
   @volatile private var error: Throwable = UNCAUGHT_ERROR
-  @volatile private var lastRowOfLog: String = "unknown"
+
+  private val engineLogMaxLines = conf.get(KyuubiConf.SESSION_ENGINE_STARTUP_MAX_LOG_LINES)
+  private val lastRowsOfLog: EvictingQueue[String] = EvictingQueue.create(engineLogMaxLines)
   // Visible for test
   @volatile private[kyuubi] var logCaptureThreadReleased: Boolean = true
   private var logCaptureThread: Thread = _
@@ -130,7 +133,7 @@ trait ProcBuilder {
 
               error = KyuubiSQLException(sb.toString() + s"\n See more: $engineLog")
             } else if (line != null) {
-              lastRowOfLog = line
+              lastRowsOfLog.add(line)
             }
           } else {
             Thread.sleep(300)
@@ -153,7 +156,7 @@ trait ProcBuilder {
 
   val YARN_APP_NAME_REGEX: Regex = "application_\\d+_\\d+".r
 
-  def killApplication(line: String = lastRowOfLog): String =
+  def killApplication(line: String = lastRowsOfLog.toArray.mkString("\n")): String =
     YARN_APP_NAME_REGEX.findFirstIn(line) match {
       case Some(appId) =>
         env.get(KyuubiConf.KYUUBI_HOME) match {
@@ -187,7 +190,8 @@ trait ProcBuilder {
     error match {
       case UNCAUGHT_ERROR =>
         KyuubiSQLException(s"Failed to detect the root cause, please check $engineLog at server " +
-          s"side if necessary. The last line log is: $lastRowOfLog")
+          s"side if necessary. The last $engineLogMaxLines line(s) of log is: " +
+          s"${lastRowsOfLog.toArray.mkString("\n")}")
       case other => other
     }
   }

kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerConnectionSuite.scala

@@ -63,7 +63,6 @@ class KyuubiOperationPerConnectionSuite extends WithKyuubiServer with JDBCTestUt
       }
       val verboseMessage = Utils.stringifyException(exception)
       assert(verboseMessage.contains("Failed to detect the root cause"))
-      assert(verboseMessage.contains("The last line log"))
     }
   }
 }