diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala index d89925081..0dbbec03f 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala @@ -41,9 +41,12 @@ object KyuubiHadoopUtils { classOf[Credentials].getDeclaredField("tokenMap") tokenMapField.setAccessible(true) - def newHadoopConf(conf: KyuubiConf): Configuration = { - val hadoopConf = new Configuration() - conf.getAll.foreach { case (k, v) => hadoopConf.set(k, v) } + def newHadoopConf( + conf: KyuubiConf, + loadDefaults: Boolean = true): Configuration = { + val hadoopConf = new Configuration(loadDefaults) + conf.getAll + .foreach { case (k, v) => hadoopConf.set(k, v) } hadoopConf } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/util/KyuubiHadoopUtilsSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/util/KyuubiHadoopUtilsSuite.scala index 9a6d76732..c22d6ea54 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/util/KyuubiHadoopUtilsSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/util/KyuubiHadoopUtilsSuite.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi.util +import java.util.stream.StreamSupport + import scala.util.Random import org.apache.hadoop.io.Text @@ -59,4 +61,18 @@ class KyuubiHadoopUtilsSuite extends KyuubiFunSuite { KyuubiHadoopUtils.encodeCredentials(credentials)) assert(decoded.getToken(token.getKind) == credentials.getToken(token.getKind)) } + + test("new hadoop conf with kyuubi conf with loadDefaults") { + val abc = "kyuubi.abc" + val kyuubiConf = new KyuubiConf() + .set(abc, "xyz") + + var hadoopConf = KyuubiHadoopUtils.newHadoopConf(kyuubiConf) + assert(StreamSupport.stream(hadoopConf.spliterator(), false) + .anyMatch(entry => entry.getKey.startsWith("hadoop") || entry.getKey.startsWith("fs"))) + + hadoopConf = KyuubiHadoopUtils.newHadoopConf(kyuubiConf, loadDefaults = false) + assert(StreamSupport.stream(hadoopConf.spliterator(), false) + .noneMatch(entry => entry.getKey.startsWith("hadoop") || entry.getKey.startsWith("fs"))) + } } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopFsDelegationTokenProvider.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopFsDelegationTokenProvider.scala index 2f6b17a99..d543a99b8 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopFsDelegationTokenProvider.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopFsDelegationTokenProvider.scala @@ -31,6 +31,7 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod import org.apache.kyuubi.Logging import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.credentials.HadoopFsDelegationTokenProvider.{disableFsCache, doAsProxyUser} +import org.apache.kyuubi.util.KyuubiHadoopUtils class HadoopFsDelegationTokenProvider extends HadoopDelegationTokenProvider with Logging { @@ -80,7 +81,10 @@ object HadoopFsDelegationTokenProvider { def disableFsCache(kyuubiConf: KyuubiConf, hadoopConf: Configuration): Configuration = { // Avoid unnecessary disk io by not loading default resources - val newConf = new Configuration(false) + val newConf = KyuubiHadoopUtils.newHadoopConf( + kyuubiConf, + loadDefaults = false) + hadoopConf.iterator().asScala.foreach(e => newConf.set(e.getKey, e.getValue)) hadoopFSsToAccess(kyuubiConf, hadoopConf) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/EventLoggingService.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/EventLoggingService.scala index 593ef7069..ee7e65e22 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/EventLoggingService.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/EventLoggingService.scala @@ -19,8 +19,6 @@ package org.apache.kyuubi.server import java.net.InetAddress -import org.apache.hadoop.conf.Configuration - import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.SERVER_EVENT_JSON_LOG_PATH import org.apache.kyuubi.config.KyuubiConf.SERVER_EVENT_LOGGERS @@ -33,6 +31,7 @@ import org.apache.kyuubi.util.KyuubiHadoopUtils class EventLoggingService extends AbstractEventLoggingService[KyuubiServerEvent] { override def initialize(conf: KyuubiConf): Unit = { + val hadoopConf = KyuubiHadoopUtils.newHadoopConf(conf) conf.get(SERVER_EVENT_LOGGERS) .map(EventLoggerType.withName) .foreach { @@ -41,9 +40,9 @@ class EventLoggingService extends AbstractEventLoggingService[KyuubiServerEvent] val jsonEventLogger = new JsonEventLogger[KyuubiServerEvent]( s"server-$hostName", SERVER_EVENT_JSON_LOG_PATH, - new Configuration()) + hadoopConf) // TODO: #1180 kyuubiServerEvent need create logRoot automatically - jsonEventLogger.createEventLogRootDir(conf, KyuubiHadoopUtils.newHadoopConf(conf)) + jsonEventLogger.createEventLogRootDir(conf, hadoopConf) addService(jsonEventLogger) addEventLogger(jsonEventLogger) case logger =>