From 16cfee2fcaa188b3c8394f79b7d24c41d39004e0 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 13 Jun 2021 22:33:52 +0800 Subject: [PATCH] [KYUUBI #687] Enable transfer env variables to engine launcher process ### _Why are the changes needed?_ Enable transfer some env variables to engine launcher process ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request Closes #687 from turboFei/spark_proc_env. Closes #687 262c320 [fwang12] address comments b2f3539 [fwang12] use kyuubi engine env instead of env list 812b0ec [fwang12] move to general e664814 [fwang12] Enable transfer env map to engine launcher process Authored-by: fwang12 Signed-off-by: Kent Yao --- .../scala/org/apache/kyuubi/config/KyuubiConf.scala | 6 ++++++ .../org/apache/kyuubi/config/internal/Tests.scala | 5 ----- .../org/apache/kyuubi/engine/ProcBuilder.scala | 2 +- .../kyuubi/engine/spark/SparkProcessBuilder.scala | 13 +------------ .../org/apache/kyuubi/WithKyuubiServerOnYarn.scala | 4 ++-- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 5dade0af6..3793a5eed 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -101,6 +101,11 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { settings.entrySet().asScala.map(x => (x.getKey, x.getValue)).toMap[String, String] } + /** Get all envs as map */ + def getEnvs: Map[String, String] = { + sys.env ++ getAllWithPrefix(KYUUBI_ENGINE_ENV_PREFIX, "") + } + /** * Retrieve key-value pairs from [[KyuubiConf]] starting with `dropped.remainder`, and put them to * the result map with the `dropped` of key being dropped. @@ -177,6 +182,7 @@ object KyuubiConf { /** the default file that contains kyuubi properties */ final val KYUUBI_CONF_FILE_NAME = "kyuubi-defaults.conf" final val KYUUBI_HOME = "KYUUBI_HOME" + final val KYUUBI_ENGINE_ENV_PREFIX = "kyuubi.engineEnv" val kyuubiConfEntries: java.util.Map[String, ConfigEntry[_]] = java.util.Collections.synchronizedMap(new java.util.HashMap[String, ConfigEntry[_]]()) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/internal/Tests.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/internal/Tests.scala index 8b8101e34..13089a642 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/internal/Tests.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/internal/Tests.scala @@ -24,9 +24,4 @@ private[kyuubi] object Tests { .version("1.2.0") .booleanConf .createOptional - - val TESTING_HADOOP_CONF_DIR = ConfigBuilder("kyuubi.testing.hadoop.conf.dir") - .version("1.3.0") - .stringConf - .createOptional } diff --git a/kyuubi-main/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala b/kyuubi-main/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala index 95bdfa7a5..ed64fa2ce 100644 --- a/kyuubi-main/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala +++ b/kyuubi-main/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala @@ -47,7 +47,7 @@ trait ProcBuilder { protected def conf: KyuubiConf - protected def env: Map[String, String] + protected def env: Map[String, String] = conf.getEnvs protected val workingDir: Path diff --git a/kyuubi-main/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-main/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index 3744640b1..4b4029c90 100644 --- a/kyuubi-main/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-main/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -28,26 +28,15 @@ import org.apache.hadoop.security.UserGroupInformation import org.apache.kyuubi._ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAIN_RESOURCE -import org.apache.kyuubi.config.internal.Tests import org.apache.kyuubi.engine.ProcBuilder class SparkProcessBuilder( override val proxyUser: String, - override val conf: KyuubiConf, - envMap: Map[String, String] = sys.env) + override val conf: KyuubiConf) extends ProcBuilder with Logging { import SparkProcessBuilder._ - override protected def env: Map[String, String] = { - val testingHadoopConfDir = conf.getOption(Tests.TESTING_HADOOP_CONF_DIR.key) - if (Utils.isTesting && testingHadoopConfDir.isDefined) { - envMap ++ Map("HADOOP_CONF_DIR" -> testingHadoopConfDir.get) - } else { - envMap - } - } - override protected val executable: String = { val sparkHomeOpt = env.get("SPARK_HOME").orElse { val kyuubiPattern = "/kyuubi/" diff --git a/kyuubi-main/src/test/scala/org/apache/kyuubi/WithKyuubiServerOnYarn.scala b/kyuubi-main/src/test/scala/org/apache/kyuubi/WithKyuubiServerOnYarn.scala index ab54b6840..fe9f03a6c 100644 --- a/kyuubi-main/src/test/scala/org/apache/kyuubi/WithKyuubiServerOnYarn.scala +++ b/kyuubi-main/src/test/scala/org/apache/kyuubi/WithKyuubiServerOnYarn.scala @@ -18,7 +18,7 @@ package org.apache.kyuubi import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.internal.Tests.TESTING_HADOOP_CONF_DIR +import org.apache.kyuubi.config.KyuubiConf.KYUUBI_ENGINE_ENV_PREFIX import org.apache.kyuubi.server.MiniYarnService import org.apache.kyuubi.util.KyuubiHadoopUtils @@ -47,7 +47,7 @@ trait WithKyuubiServerOnYarn extends WithKyuubiServer { KyuubiHadoopUtils.toSparkPrefixedConf(miniYarnService.getHadoopConf()).foreach { case (k, v) => conf.set(k, v) } - conf.set(TESTING_HADOOP_CONF_DIR, miniYarnService.getHadoopConfDir()) + conf.set(s"$KYUUBI_ENGINE_ENV_PREFIX.HADOOP_CONF_DIR", miniYarnService.getHadoopConfDir) super.beforeAll() }