diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala index 6e7a3168a..e2cf3adf9 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala @@ -24,10 +24,10 @@ import scala.util.control.NonFatal import org.apache.spark.{ui, SparkConf} import org.apache.spark.kyuubi.SparkSQLEngineListener -import org.apache.spark.repl.Main +import org.apache.spark.kyuubi.SparkUtilsHelper.getLocalDir import org.apache.spark.sql.SparkSession -import org.apache.kyuubi.{KyuubiException, Logging} +import org.apache.kyuubi.{KyuubiException, Logging, Utils} import org.apache.kyuubi.Utils._ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf._ @@ -78,13 +78,15 @@ object SparkSQLEngine extends Logging { def createSpark(): SparkSession = { val sparkConf = new SparkConf() + val rootDir = sparkConf.getOption("spark.repl.classdir").getOrElse(getLocalDir(sparkConf)) + val outputDir = Utils.createTempDir(root = rootDir, namePrefix = "repl") sparkConf.setIfMissing("spark.sql.execution.topKSortFallbackThreshold", "10000") sparkConf.setIfMissing("spark.sql.legacy.castComplexTypesToString.enabled", "true") sparkConf.setIfMissing("spark.master", "local") sparkConf.setIfMissing("spark.ui.port", "0") // register the repl's output dir with the file server. // see also `spark.repl.classdir` - sparkConf.set("spark.repl.class.outputDir", Main.outputDir.getAbsolutePath) + sparkConf.set("spark.repl.class.outputDir", outputDir.toFile.getAbsolutePath) sparkConf.setIfMissing( "spark.hadoop.mapreduce.input.fileinputformat.list-status.num-threads", "20") diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala index a0c6df58e..c5f25a834 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala @@ -24,7 +24,7 @@ import scala.tools.nsc.interpreter.IR import scala.tools.nsc.interpreter.JPrintWriter import org.apache.spark.SparkContext -import org.apache.spark.repl.{Main, SparkILoop} +import org.apache.spark.repl.SparkILoop import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.util.MutableURLClassLoader @@ -40,7 +40,7 @@ private[spark] case class KyuubiSparkILoop private ( val interpArguments = List( "-Yrepl-class-based", "-Yrepl-outdir", - s"${Main.outputDir.getAbsolutePath}") + s"${spark.sparkContext.getConf.get("spark.repl.class.outputDir")}") settings.processArguments(interpArguments, processAll = true) settings.usejavacp.value = true val currentClassLoader = Thread.currentThread().getContextClassLoader diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkUtilsHelper.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkUtilsHelper.scala index 4af400221..e2f51e648 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkUtilsHelper.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkUtilsHelper.scala @@ -19,6 +19,7 @@ package org.apache.spark.kyuubi import scala.util.matching.Regex +import org.apache.spark.SparkConf import org.apache.spark.util.Utils import org.apache.kyuubi.Logging @@ -35,4 +36,11 @@ object SparkUtilsHelper extends Logging { def redact(regex: Option[Regex], text: String): String = { Utils.redact(regex, text) } + + /** + * Get the path of a temporary directory. + */ + def getLocalDir(conf: SparkConf): String = { + Utils.getLocalDir(conf) + } }