[KYUUBI #5009] Pass Spark Engine Log Path to Spark Conf

### _Why are the changes needed?_

Close #5009

When Kyuubi Server Log is Huge, it's difficult to find `Spark Engine Log Path` in logs.

Here pass the path to spark conf, user can find engine log path in spark ui or spark history server.

Submit Command Like:
```shell
XXXX/bin/spark-submit \
  --class org.apache.kyuubi.engine.spark.SparkSQLEngine \
  --conf spark.kyuubi.engine.engineLog.path=XXXX/kyuubi-spark-sql-engine.log.0 \
  --proxy-user kyuubi XXXX/target/kyuubi-spark-sql-engine_2.12-1.8.0-SNAPSHOT.jar
```

### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request

Closes #5011 from zwangsheng/KYUUBI_5009.

Closes #5009

36c772209 [zwangsheng] fix compile
1c20f9264 [zwangsheng] retest
70568c758 [zwangsheng] Fix Unit Test
2bc465740 [zwangsheng] try to fix unit test
2197b3503 [zwangsheng] Narrow the scope of access
a44eefc5c [zwangsheng] [KYUUBI #5009]Pass Spark Engine Log Path to Spark COnf

Authored-by: zwangsheng <2213335496@qq.com>
Signed-off-by: Cheng Pan <chengpan@apache.org>

kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala

@@ -359,4 +359,6 @@ object ProcBuilder extends Logging {
   private val PROC_BUILD_LOGGER = new NamedThreadFactory("process-logger-capture", daemon = true)
 
   private val UNCAUGHT_ERROR = new RuntimeException("Uncaught error")
+
+  private[engine] val KYUUBI_ENGINE_LOG_PATH_KEY = "kyuubi.engine.engineLog.path"
 }

kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala

@@ -36,7 +36,7 @@ class SparkBatchProcessBuilder(
   extends SparkProcessBuilder(proxyUser, conf, batchId, extraEngineLog) {
   import SparkProcessBuilder._
 
-  override protected val commands: Array[String] = {
+  override protected lazy val commands: Array[String] = {
     val buffer = new ArrayBuffer[String]()
     buffer += executable
     Option(mainClass).foreach { cla =>
@@ -51,7 +51,7 @@ class SparkBatchProcessBuilder(
     // tag batch application
     KyuubiApplicationManager.tagApplication(batchId, "spark", clusterManager(), batchKyuubiConf)
 
-    (batchKyuubiConf.getAll ++ sparkAppNameConf()).foreach { case (k, v) =>
+    (batchKyuubiConf.getAll ++ sparkAppNameConf() ++ engineLogPathConf()).foreach { case (k, v) =>
       buffer += CONF
       buffer += s"${convertConfigKey(k)}=$v"
     }

kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala

@@ -30,6 +30,7 @@ import org.apache.kyuubi._
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.engine.{ApplicationManagerInfo, KyuubiApplicationManager, ProcBuilder}
 import org.apache.kyuubi.engine.KubernetesApplicationOperation.{KUBERNETES_SERVICE_HOST, KUBERNETES_SERVICE_PORT}
+import org.apache.kyuubi.engine.ProcBuilder.KYUUBI_ENGINE_LOG_PATH_KEY
 import org.apache.kyuubi.ha.HighAvailabilityConf
 import org.apache.kyuubi.ha.client.AuthTypes
 import org.apache.kyuubi.operation.log.OperationLog
@@ -99,7 +100,7 @@ class SparkProcessBuilder(
     }
   }
 
-  override protected val commands: Array[String] = {
+  override protected lazy val commands: Array[String] = {
     // complete `spark.master` if absent on kubernetes
     completeMasterUrl(conf)
 
@@ -116,8 +117,8 @@ class SparkProcessBuilder(
         == AuthTypes.KERBEROS) {
       allConf = allConf ++ zkAuthKeytabFileConf(allConf)
     }
-
-    allConf.foreach { case (k, v) =>
+    // pass spark engine log path to spark conf
+    (allConf ++ engineLogPathConf).foreach { case (k, v) =>
       buffer += CONF
       buffer += s"${convertConfigKey(k)}=$v"
     }
@@ -244,6 +245,10 @@ class SparkProcessBuilder(
       }
     }
   }
+
+  private[spark] def engineLogPathConf(): Map[String, String] = {
+    Map(KYUUBI_ENGINE_LOG_PATH_KEY -> engineLog.getAbsolutePath)
+  }
 }
 
 object SparkProcessBuilder {

kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala

@@ -29,6 +29,8 @@ import org.scalatestplus.mockito.MockitoSugar
 import org.apache.kyuubi.{KerberizedTestHelper, KyuubiSQLException, Utils}
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.config.KyuubiConf.{ENGINE_LOG_TIMEOUT, ENGINE_SPARK_MAIN_RESOURCE}
+import org.apache.kyuubi.engine.ProcBuilder.KYUUBI_ENGINE_LOG_PATH_KEY
+import org.apache.kyuubi.engine.spark.SparkProcessBuilder.CONF
 import org.apache.kyuubi.ha.HighAvailabilityConf
 import org.apache.kyuubi.ha.client.AuthTypes
 import org.apache.kyuubi.service.ServiceUtils
@@ -296,9 +298,16 @@ class SparkProcessBuilderSuite extends KerberizedTestHelper with MockitoSugar {
     assert(!c3.contains(s"spark.kubernetes.driverEnv.SPARK_USER_NAME=$proxyName"))
     assert(!c3.contains(s"spark.executorEnv.SPARK_USER_NAME=$proxyName"))
   }
+
+  test("[KYUUBI #5009] Test pass spark engine log path to spark conf") {
+    val b1 = new SparkProcessBuilder("kyuubi", conf)
+    assert(
+      b1.toString.contains(
+        s"$CONF spark.$KYUUBI_ENGINE_LOG_PATH_KEY=${b1.engineLog.getAbsolutePath}"))
+  }
 }
 
 class FakeSparkProcessBuilder(config: KyuubiConf)
   extends SparkProcessBuilder("fake", config) {
-  override protected val commands: Array[String] = Array("ls")
+  override protected lazy val commands: Array[String] = Array("ls")
 }