[KYUUBI #687] Enable transfer env variables to engine launcher process

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/NetEase/kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
Enable transfer some env variables to engine launcher process

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request

Closes #687 from turboFei/spark_proc_env.

Closes #687

262c320 [fwang12] address comments
b2f3539 [fwang12] use kyuubi engine env instead of env list
812b0ec [fwang12] move to general
e664814 [fwang12] Enable transfer env map to engine launcher process

Authored-by: fwang12 <fwang12@ebay.com>
Signed-off-by: Kent Yao <yao@apache.org>

kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

@@ -101,6 +101,11 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
     settings.entrySet().asScala.map(x => (x.getKey, x.getValue)).toMap[String, String]
   }
 
+  /** Get all envs as map */
+  def getEnvs: Map[String, String] = {
+    sys.env ++ getAllWithPrefix(KYUUBI_ENGINE_ENV_PREFIX, "")
+  }
+
   /**
    * Retrieve key-value pairs from [[KyuubiConf]] starting with `dropped.remainder`, and put them to
    * the result map with the `dropped` of key being dropped.
@@ -177,6 +182,7 @@ object KyuubiConf {
   /** the default file that contains kyuubi properties */
   final val KYUUBI_CONF_FILE_NAME = "kyuubi-defaults.conf"
   final val KYUUBI_HOME = "KYUUBI_HOME"
+  final val KYUUBI_ENGINE_ENV_PREFIX = "kyuubi.engineEnv"
 
   val kyuubiConfEntries: java.util.Map[String, ConfigEntry[_]] =
     java.util.Collections.synchronizedMap(new java.util.HashMap[String, ConfigEntry[_]]())

kyuubi-common/src/main/scala/org/apache/kyuubi/config/internal/Tests.scala

@@ -24,9 +24,4 @@ private[kyuubi] object Tests {
     .version("1.2.0")
     .booleanConf
     .createOptional
-
-  val TESTING_HADOOP_CONF_DIR = ConfigBuilder("kyuubi.testing.hadoop.conf.dir")
-    .version("1.3.0")
-    .stringConf
-    .createOptional
 }

kyuubi-main/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala

@@ -47,7 +47,7 @@ trait ProcBuilder {
 
   protected def conf: KyuubiConf
 
-  protected def env: Map[String, String]
+  protected def env: Map[String, String] = conf.getEnvs
 
   protected val workingDir: Path
 

kyuubi-main/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala

@@ -28,26 +28,15 @@ import org.apache.hadoop.security.UserGroupInformation
 import org.apache.kyuubi._
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAIN_RESOURCE
-import org.apache.kyuubi.config.internal.Tests
 import org.apache.kyuubi.engine.ProcBuilder
 
 class SparkProcessBuilder(
     override val proxyUser: String,
-    override val conf: KyuubiConf,
+    override val conf: KyuubiConf)
-    envMap: Map[String, String] = sys.env)
   extends ProcBuilder with Logging {
 
   import SparkProcessBuilder._
 
-  override protected def env: Map[String, String] = {
-    val testingHadoopConfDir = conf.getOption(Tests.TESTING_HADOOP_CONF_DIR.key)
-    if (Utils.isTesting && testingHadoopConfDir.isDefined) {
-      envMap ++ Map("HADOOP_CONF_DIR" -> testingHadoopConfDir.get)
-    } else {
-      envMap
-    }
-  }
-
   override protected val executable: String = {
     val sparkHomeOpt = env.get("SPARK_HOME").orElse {
       val kyuubiPattern = "/kyuubi/"

kyuubi-main/src/test/scala/org/apache/kyuubi/WithKyuubiServerOnYarn.scala

@@ -18,7 +18,7 @@
 package org.apache.kyuubi
 
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.internal.Tests.TESTING_HADOOP_CONF_DIR
+import org.apache.kyuubi.config.KyuubiConf.KYUUBI_ENGINE_ENV_PREFIX
 import org.apache.kyuubi.server.MiniYarnService
 import org.apache.kyuubi.util.KyuubiHadoopUtils
 
@@ -47,7 +47,7 @@ trait WithKyuubiServerOnYarn extends WithKyuubiServer {
     KyuubiHadoopUtils.toSparkPrefixedConf(miniYarnService.getHadoopConf()).foreach { case (k, v) =>
       conf.set(k, v)
     }
-    conf.set(TESTING_HADOOP_CONF_DIR, miniYarnService.getHadoopConfDir())
+    conf.set(s"$KYUUBI_ENGINE_ENV_PREFIX.HADOOP_CONF_DIR", miniYarnService.getHadoopConfDir)
     super.beforeAll()
   }