apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #1786] Fix the Spark sql engine logger level changed to WARN issue

Browse Source 
<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->

To close #1786
Main object will init logger with isInterpreter=true, which changes logger to WARN level.
In this pr, we do not use Main.output as `spark.repl.class.outputDir`
### _How was this patch tested?_
Passed existing tests.

Closes #1828 from turboFei/repl_warn.

Closes #1786

91fc327c [Fei Wang] address commments
762d148c [Fei Wang] refactor
4e18f367 [Fei Wang] do not use main output dir

Authored-by: Fei Wang <fwang12@ebay.com>
Signed-off-by: Fei Wang <fwang12@ebay.com>
This commit is contained in:
Fei Wang 2022-01-25 15:26:19 +08:00
parent 68f70fd005
commit c29c4430aa
3 changed files with 15 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala vendored
View File

@ -24,10 +24,10 @@ import scala.util.control.NonFatal
import org.apache.spark.{ui, SparkConf}
import org.apache.spark.kyuubi.SparkSQLEngineListener
import org.apache.spark.repl.Main
import org.apache.spark.kyuubi.SparkUtilsHelper.getLocalDir
import org.apache.spark.sql.SparkSession
import org.apache.kyuubi.{KyuubiException, Logging}
import org.apache.kyuubi.{KyuubiException, Logging, Utils}
import org.apache.kyuubi.Utils._
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf._
@ -78,13 +78,15 @@ object SparkSQLEngine extends Logging {
def createSpark(): SparkSession = {
val sparkConf = new SparkConf()
val rootDir = sparkConf.getOption("spark.repl.classdir").getOrElse(getLocalDir(sparkConf))
val outputDir = Utils.createTempDir(root = rootDir, namePrefix = "repl")
sparkConf.setIfMissing("spark.sql.execution.topKSortFallbackThreshold", "10000")
sparkConf.setIfMissing("spark.sql.legacy.castComplexTypesToString.enabled", "true")
sparkConf.setIfMissing("spark.master", "local")
sparkConf.setIfMissing("spark.ui.port", "0")
// register the repl's output dir with the file server.
// see also `spark.repl.classdir`
sparkConf.set("spark.repl.class.outputDir", Main.outputDir.getAbsolutePath)
sparkConf.set("spark.repl.class.outputDir", outputDir.toFile.getAbsolutePath)
sparkConf.setIfMissing(
"spark.hadoop.mapreduce.input.fileinputformat.list-status.num-threads",
"20")

4
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala vendored
View File

@ -24,7 +24,7 @@ import scala.tools.nsc.interpreter.IR
import scala.tools.nsc.interpreter.JPrintWriter
import org.apache.spark.SparkContext
import org.apache.spark.repl.{Main, SparkILoop}
import org.apache.spark.repl.SparkILoop
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.util.MutableURLClassLoader
@ -40,7 +40,7 @@ private[spark] case class KyuubiSparkILoop private (
val interpArguments = List(
"-Yrepl-class-based",
"-Yrepl-outdir",
s"${Main.outputDir.getAbsolutePath}")
s"${spark.sparkContext.getConf.get("spark.repl.class.outputDir")}")
settings.processArguments(interpArguments, processAll = true)
settings.usejavacp.value = true
val currentClassLoader = Thread.currentThread().getContextClassLoader

8
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkUtilsHelper.scala vendored
View File

@ -19,6 +19,7 @@ package org.apache.spark.kyuubi
import scala.util.matching.Regex
import org.apache.spark.SparkConf
import org.apache.spark.util.Utils
import org.apache.kyuubi.Logging
@ -35,4 +36,11 @@ object SparkUtilsHelper extends Logging {
def redact(regex: Option[Regex], text: String): String = {
Utils.redact(regex, text)
}
/**
* Get the path of a temporary directory.
*/
def getLocalDir(conf: SparkConf): String = {
Utils.getLocalDir(conf)
}
}