apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

fix work dir bug

Browse Source
This commit is contained in:
Kent Yao 2020-10-14 14:59:49 +08:00
parent 0b07dfd6dd
commit e7a1054fd2
4 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala vendored
View File

@ -39,7 +39,6 @@ private[spark] final class SparkSQLEngine(name: String, spark: SparkSession)
override protected def stopServer(): Unit = spark.stop()
}
object SparkSQLEngine extends Logging {
val kyuubiConf: KyuubiConf = KyuubiConf()

3
kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala
View File

@ -98,7 +98,8 @@ class ServiceDiscovery private (
.forPath(s"/$namespace")
} catch {
case _: NodeExistsException => // do nothing
case e: KeeperException => throw new KyuubiException(s"Failed to create namespace '/$namespace'", e)
case e: KeeperException =>
throw new KyuubiException(s"Failed to create namespace '/$namespace'", e)
}
super.initialize(conf)
}

9
kyuubi-main/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
View File

@ -20,7 +20,6 @@ package org.apache.kyuubi.engine
import java.io.IOException
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path, Paths}
import java.util.UUID
import scala.collection.JavaConverters._
@ -34,6 +33,8 @@ trait ProcBuilder {
protected def mainResource: Option[String]
protected def module: String
protected def mainClass: String
protected def proxyUser: String
@ -42,7 +43,7 @@ trait ProcBuilder {
protected def env: Map[String, String]
protected def workingDir: Path
protected val workingDir: Path
final lazy val processBuilder: ProcessBuilder = {
val pb = new ProcessBuilder(commands: _*)
@ -56,7 +57,7 @@ trait ProcBuilder {
private var error: Throwable = UNCAUGHT_ERROR
final def start: Process = {
val procLog = Paths.get(workingDir.toAbsolutePath.toString, UUID.randomUUID().toString)
val procLog = Paths.get(workingDir.toAbsolutePath.toString, s"$module.log")
processBuilder.redirectError(procLog.toFile)
processBuilder.redirectOutput(procLog.toFile)
@ -98,7 +99,7 @@ trait ProcBuilder {
}
object ProcBuilder {
private val PROC_BUILD_LOGGER = NamedThreadFactory("process-logger", daemon = true)
private val PROC_BUILD_LOGGER = NamedThreadFactory("process-logger-capture", daemon = true)
private val UNCAUGHT_ERROR = KyuubiSQLException("Uncaught error")

15
kyuubi-main/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
View File

@ -56,7 +56,7 @@ class SparkProcessBuilder(
override def mainResource: Option[String] = {
// 1. get the main resource jar for user specified config first
val jarName = s"kyuubi-spark-sql-engine-$KYUUBI_VERSION.jar"
val jarName = s"$module-$KYUUBI_VERSION.jar"
conf.get(ENGINE_SPARK_MAIN_RESOURCE.key).filter { userSpecified =>
Files.exists(Paths.get(userSpecified))
}.orElse {
@ -66,18 +66,18 @@ class SparkProcessBuilder(
.filter(Files.exists(_)).map(_.toAbsolutePath.toFile.getCanonicalPath)
}.orElse {
// 3. get the main resource from dev environment
Option(Paths.get("externals", "kyuubi-spark-sql-engine", "target", jarName))
Option(Paths.get("externals", module, "target", jarName))
.filter(Files.exists(_)).orElse {
Some(Paths.get("..", "externals", "kyuubi-spark-sql-engine", "target", jarName))
Some(Paths.get("..", "externals", module, "target", jarName))
}.map(_.toAbsolutePath.toFile.getCanonicalPath)
}
}
override protected def workingDir: Path = {
override protected val workingDir: Path = {
env.get("KYUUBI_WORK_DIR_ROOT").map { root =>
Utils.createDirectory(root, proxyUser)
Utils.createTempDir(root, proxyUser)
}.getOrElse {
Utils.createDirectory(System.getProperty("java.io.tmpdir"), proxyUser)
Utils.createTempDir(proxyUser)
}
}
@ -99,8 +99,9 @@ class SparkProcessBuilder(
}
override def toString: String = commands.mkString(" ")
}
override protected def module: String = "kyuubi-spark-sql-engine"
}
object SparkProcessBuilder {
private final val CONF = "--conf"