From 28421ec5c13eb73d42dbbdffc092f02da0922434 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 May 2018 19:51:11 +0800 Subject: [PATCH] info -> debug logging --- .../v2.1/scala/org/apache/spark/SparkEnv.scala | 8 ++++---- .../v2.2/scala/org/apache/spark/SparkEnv.scala | 4 ++++ .../v2.3/scala/org/apache/spark/SparkEnv.scala | 4 ++++ .../scala/org/apache/spark/KyuubiSparkUtil.scala | 13 ------------- .../scala/yaooqinn/kyuubi/auth/KerberosTest.scala | 3 +-- 5 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/extra/v2.1/scala/org/apache/spark/SparkEnv.scala b/src/extra/v2.1/scala/org/apache/spark/SparkEnv.scala index 9250a5da4..b4cfe4448 100644 --- a/src/extra/v2.1/scala/org/apache/spark/SparkEnv.scala +++ b/src/extra/v2.1/scala/org/apache/spark/SparkEnv.scala @@ -137,7 +137,7 @@ class SparkEnv ( } object SparkEnv extends Logging { - logInfo("Kyuubi Supplied SparkEnv Class") + logInfo("Loaded Kyuubi Supplied SparkEnv Class...") private val env = new ConcurrentHashMap[String, SparkEnv]() private[spark] val driverSystemName = "sparkDriver" @@ -147,10 +147,10 @@ object SparkEnv extends Logging { def set(e: SparkEnv) { if (e == null) { - logInfo(s"Removing SparkEnv for $user") + logDebug(s"Kyuubi: Removing SparkEnv for $user") env.remove(user) } else { - logInfo(s"Registering SparkEnv for $user") + logDebug(s"Kyuubi: Registering SparkEnv for $user") env.put(user, e) } } @@ -159,7 +159,7 @@ object SparkEnv extends Logging { * Returns the SparkEnv. */ def get: SparkEnv = { - logInfo(s"Get SparkEnv for $user") + logDebug(s"Kyuubi: Get SparkEnv for $user") env.get(user) } diff --git a/src/extra/v2.2/scala/org/apache/spark/SparkEnv.scala b/src/extra/v2.2/scala/org/apache/spark/SparkEnv.scala index 7f305851f..38f91660c 100644 --- a/src/extra/v2.2/scala/org/apache/spark/SparkEnv.scala +++ b/src/extra/v2.2/scala/org/apache/spark/SparkEnv.scala @@ -138,6 +138,7 @@ class SparkEnv ( } object SparkEnv extends Logging { + logInfo("Loaded Kyuubi Supplied SparkEnv Class...") private val env = new ConcurrentHashMap[String, SparkEnv]() private[spark] val driverSystemName = "sparkDriver" @@ -147,8 +148,10 @@ object SparkEnv extends Logging { def set(e: SparkEnv) { if (e == null) { + logDebug(s"Kyuubi: Removing SparkEnv for $user") env.remove(user) } else { + logDebug(s"Kyuubi: Registering SparkEnv for $user") env.put(user, e) } } @@ -157,6 +160,7 @@ object SparkEnv extends Logging { * Returns the SparkEnv. */ def get: SparkEnv = { + logDebug(s"Kyuubi: Get SparkEnv for $user") env.get(user) } diff --git a/src/extra/v2.3/scala/org/apache/spark/SparkEnv.scala b/src/extra/v2.3/scala/org/apache/spark/SparkEnv.scala index 6b0d2413e..99fc6293b 100644 --- a/src/extra/v2.3/scala/org/apache/spark/SparkEnv.scala +++ b/src/extra/v2.3/scala/org/apache/spark/SparkEnv.scala @@ -138,6 +138,7 @@ class SparkEnv ( } object SparkEnv extends Logging { + logInfo("Loaded Kyuubi Supplied SparkEnv Class...") private val env = new ConcurrentHashMap[String, SparkEnv]() private[spark] val driverSystemName = "sparkDriver" @@ -147,8 +148,10 @@ object SparkEnv extends Logging { def set(e: SparkEnv) { if (e == null) { + logDebug(s"Kyuubi: Removing SparkEnv for $user") env.remove(user) } else { + logDebug(s"Kyuubi: Registering SparkEnv for $user") env.put(user, e) } } @@ -157,6 +160,7 @@ object SparkEnv extends Logging { * Returns the SparkEnv. */ def get: SparkEnv = { + logDebug(s"Kyuubi: Get SparkEnv for $user") env.get(user) } diff --git a/src/main/scala/org/apache/spark/KyuubiSparkUtil.scala b/src/main/scala/org/apache/spark/KyuubiSparkUtil.scala index 987cf92ab..b1de3196b 100644 --- a/src/main/scala/org/apache/spark/KyuubiSparkUtil.scala +++ b/src/main/scala/org/apache/spark/KyuubiSparkUtil.scala @@ -93,29 +93,16 @@ object KyuubiSparkUtil extends Logging { Utils.getContextOrSparkClassLoader } - def getLocalDir(conf: SparkConf): String = { - Utils.getLocalDir(conf) - } - def createTempDir( root: String = System.getProperty("java.io.tmpdir"), namePrefix: String = "spark"): File = { Utils.createTempDir(root, namePrefix) } - def getUserJars(conf: SparkConf): Seq[String] = { - Utils.getUserJars(conf) - } - def newConfiguration(conf: SparkConf): Configuration = { SparkHadoopUtil.get.newConfiguration(conf) } - /** Executes the given block. Log non-fatal errors if any, and only throw fatal errors */ - def tryLogNonFatalError(block: => Unit): Unit = { - Utils.tryLogNonFatalError(block) - } - def localHostName(): String = Utils.localHostName() // org.apache.spark.util.VersionUtils: Utilities for working with Spark version strings diff --git a/src/test/scala/yaooqinn/kyuubi/auth/KerberosTest.scala b/src/test/scala/yaooqinn/kyuubi/auth/KerberosTest.scala index 8d2b0acc9..58e1de1dc 100644 --- a/src/test/scala/yaooqinn/kyuubi/auth/KerberosTest.scala +++ b/src/test/scala/yaooqinn/kyuubi/auth/KerberosTest.scala @@ -19,14 +19,13 @@ package yaooqinn.kyuubi.auth import java.io.{File, IOException} -import java.util.UUID import org.apache.hadoop.hive.shims.ShimLoader import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge import org.apache.hadoop.minikdc.MiniKdc import org.apache.hadoop.security.UserGroupInformation import org.apache.hive.service.cli.thrift.TCLIService.Client -import org.apache.spark.{KyuubiConf, SparkConf, SparkFunSuite, KyuubiSparkUtil} +import org.apache.spark.{KyuubiConf, KyuubiSparkUtil, SparkConf, SparkFunSuite} import org.apache.spark.deploy.SparkHadoopUtil import org.apache.thrift.protocol.TBinaryProtocol import org.apache.thrift.transport.TSocket