From 628719fd60f1dc2fc127c3d1b263579b0189d5eb Mon Sep 17 00:00:00 2001 From: xifeng yang Date: Wed, 5 Jan 2022 15:07:50 +0800 Subject: [PATCH] =?UTF-8?q?[KYUUBI=20#1477]=20Use=20KyuubiHadoopUtils.newH?= =?UTF-8?q?adoopConf=20instead=20of=20new=20Con=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …figuration() ### _Why are the changes needed?_ ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [x] Add screenshots for manual tests if appropriate - EventLoggingServiceSuite ![image](https://user-images.githubusercontent.com/12961802/147745991-fcb14907-4d3a-4bce-a492-1b111b6e1614.png) - HadoopFsDelegationTokenProviderSuite ![image](https://user-images.githubusercontent.com/12961802/147746114-2d2037d7-fe72-4e85-8db5-7b3244136c5b.png) - KyuubiHadoopUtilsSuite ![image](https://user-images.githubusercontent.com/12961802/147746199-40e0ca91-9f48-4395-941b-a08affb6fbd9.png) - [x] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1661 from xifeng/kyuubi-1477. Closes #1477 ad068c6e [xifeng yang] [KYUUBI #1477] Use KyuubiHadoopUtils.newHadoopConf instead of new Configuration() Authored-by: xifeng yang Signed-off-by: Kent Yao --- .../apache/kyuubi/util/KyuubiHadoopUtils.scala | 9 ++++++--- .../kyuubi/util/KyuubiHadoopUtilsSuite.scala | 16 ++++++++++++++++ .../HadoopFsDelegationTokenProvider.scala | 6 +++++- .../kyuubi/server/EventLoggingService.scala | 7 +++---- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala index d89925081..0dbbec03f 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala @@ -41,9 +41,12 @@ object KyuubiHadoopUtils { classOf[Credentials].getDeclaredField("tokenMap") tokenMapField.setAccessible(true) - def newHadoopConf(conf: KyuubiConf): Configuration = { - val hadoopConf = new Configuration() - conf.getAll.foreach { case (k, v) => hadoopConf.set(k, v) } + def newHadoopConf( + conf: KyuubiConf, + loadDefaults: Boolean = true): Configuration = { + val hadoopConf = new Configuration(loadDefaults) + conf.getAll + .foreach { case (k, v) => hadoopConf.set(k, v) } hadoopConf } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/util/KyuubiHadoopUtilsSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/util/KyuubiHadoopUtilsSuite.scala index 9a6d76732..c22d6ea54 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/util/KyuubiHadoopUtilsSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/util/KyuubiHadoopUtilsSuite.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi.util +import java.util.stream.StreamSupport + import scala.util.Random import org.apache.hadoop.io.Text @@ -59,4 +61,18 @@ class KyuubiHadoopUtilsSuite extends KyuubiFunSuite { KyuubiHadoopUtils.encodeCredentials(credentials)) assert(decoded.getToken(token.getKind) == credentials.getToken(token.getKind)) } + + test("new hadoop conf with kyuubi conf with loadDefaults") { + val abc = "kyuubi.abc" + val kyuubiConf = new KyuubiConf() + .set(abc, "xyz") + + var hadoopConf = KyuubiHadoopUtils.newHadoopConf(kyuubiConf) + assert(StreamSupport.stream(hadoopConf.spliterator(), false) + .anyMatch(entry => entry.getKey.startsWith("hadoop") || entry.getKey.startsWith("fs"))) + + hadoopConf = KyuubiHadoopUtils.newHadoopConf(kyuubiConf, loadDefaults = false) + assert(StreamSupport.stream(hadoopConf.spliterator(), false) + .noneMatch(entry => entry.getKey.startsWith("hadoop") || entry.getKey.startsWith("fs"))) + } } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopFsDelegationTokenProvider.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopFsDelegationTokenProvider.scala index 2f6b17a99..d543a99b8 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopFsDelegationTokenProvider.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopFsDelegationTokenProvider.scala @@ -31,6 +31,7 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod import org.apache.kyuubi.Logging import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.credentials.HadoopFsDelegationTokenProvider.{disableFsCache, doAsProxyUser} +import org.apache.kyuubi.util.KyuubiHadoopUtils class HadoopFsDelegationTokenProvider extends HadoopDelegationTokenProvider with Logging { @@ -80,7 +81,10 @@ object HadoopFsDelegationTokenProvider { def disableFsCache(kyuubiConf: KyuubiConf, hadoopConf: Configuration): Configuration = { // Avoid unnecessary disk io by not loading default resources - val newConf = new Configuration(false) + val newConf = KyuubiHadoopUtils.newHadoopConf( + kyuubiConf, + loadDefaults = false) + hadoopConf.iterator().asScala.foreach(e => newConf.set(e.getKey, e.getValue)) hadoopFSsToAccess(kyuubiConf, hadoopConf) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/EventLoggingService.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/EventLoggingService.scala index 593ef7069..ee7e65e22 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/EventLoggingService.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/EventLoggingService.scala @@ -19,8 +19,6 @@ package org.apache.kyuubi.server import java.net.InetAddress -import org.apache.hadoop.conf.Configuration - import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.SERVER_EVENT_JSON_LOG_PATH import org.apache.kyuubi.config.KyuubiConf.SERVER_EVENT_LOGGERS @@ -33,6 +31,7 @@ import org.apache.kyuubi.util.KyuubiHadoopUtils class EventLoggingService extends AbstractEventLoggingService[KyuubiServerEvent] { override def initialize(conf: KyuubiConf): Unit = { + val hadoopConf = KyuubiHadoopUtils.newHadoopConf(conf) conf.get(SERVER_EVENT_LOGGERS) .map(EventLoggerType.withName) .foreach { @@ -41,9 +40,9 @@ class EventLoggingService extends AbstractEventLoggingService[KyuubiServerEvent] val jsonEventLogger = new JsonEventLogger[KyuubiServerEvent]( s"server-$hostName", SERVER_EVENT_JSON_LOG_PATH, - new Configuration()) + hadoopConf) // TODO: #1180 kyuubiServerEvent need create logRoot automatically - jsonEventLogger.createEventLogRootDir(conf, KyuubiHadoopUtils.newHadoopConf(conf)) + jsonEventLogger.createEventLogRootDir(conf, hadoopConf) addService(jsonEventLogger) addEventLogger(jsonEventLogger) case logger =>