apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI #408] Fix engine log does not be overwrite

Browse Source 
![ulysses-you](https://badgen.net/badge/Hello/ulysses-you/green) [![Closes #408](https://badgen.net/badge/Preview/Closes%20%23408/blue)](https://github.com/yaooqinn/kyuubi/pull/408) ![40](https://badgen.net/badge/%2B/40/red) ![7](https://badgen.net/badge/-/7/green) ![3](https://badgen.net/badge/commits/3/yellow) ![Target Issue](https://badgen.net/badge/Missing/Target%20Issue/ff0000) ![Test Plan](https://badgen.net/badge/Missing/Test%20Plan/ff0000) ![Bug](https://badgen.net/badge/Label/Bug/) [&#10088;?&#10089;](https://pullrequestbadge.com/?utm_medium=github&utm_source=yaooqinn&utm_campaign=badge_info)<!-- PR-BADGE: PLEASE DO NOT REMOVE THIS COMMENT -->

<!--
Thanks for sending a pull request!

Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
  2. If the PR is related to an issue in https://github.com/yaooqinn/kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->

### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
  1. If you add a feature, you can talk about the use case of it.
  2. If you fix a bug, you can clarify why it is a bug.
-->
Bug fix, otherwise the engine log will always increase size with append mode.

### _How was this patch tested?_
Add new test.

Closes #408 from ulysses-you/fix-engine-log-overwrite.

7e4fd1d [ulysses-you] simply code
b8d92ee [ulysses-you] config
821891c [ulysses-you] init

Authored-by: ulysses-you <ulyssesyou18@gmail.com>
Signed-off-by: ulysses-you <ulyssesyou18@gmail.com>
(cherry picked from commit 57ed76f48d)
Signed-off-by: ulysses-you <ulyssesyou18@gmail.com>
This commit is contained in:
ulysses-you 2021-03-09 16:29:35 +08:00
parent 7107666f1b
commit 5e551bdd71
2 changed files with 40 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
kyuubi-main/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
View File

@ -65,15 +65,11 @@ trait ProcBuilder {
// Visible for test
private[kyuubi] var logCaptureThread: Thread = _
private lazy val engineLog: File = ProcBuilder.synchronized {
private[kyuubi] lazy val engineLog: File = ProcBuilder.synchronized {
val engineLogTimeout = conf.get(KyuubiConf.ENGINE_LOG_TIMEOUT)
val currentTime = System.currentTimeMillis()
val processLogPath = workingDir
val totalExistsFile = processLogPath.toFile.listFiles(new FilenameFilter() {
override def accept(dir: File, name: String): Boolean = {
name.startsWith(module)
}
})
val totalExistsFile = processLogPath.toFile.listFiles { (_, name) => name.startsWith(module) }
val sorted = totalExistsFile.sortBy(_.getName.split("\\.").last.toInt)
val nextIndex = if (sorted.isEmpty) {
0
@ -81,6 +77,18 @@ trait ProcBuilder {
sorted.last.getName.split("\\.").last.toInt + 1
}
val file = sorted.find(_.lastModified() < currentTime - engineLogTimeout)
.map { existsFile =>
try {
// Here we want to overwrite the exists log file
existsFile.delete()
existsFile.createNewFile()
existsFile
} catch {
case e: Exception =>
warn(s"failed to delete engine log file: ${existsFile.getAbsolutePath}", e)
null
}
}
.getOrElse {
Files.createDirectories(processLogPath)
val newLogFile = new File(processLogPath.toFile, s"$module.log.$nextIndex")

27
kyuubi-main/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
View File

@ -18,13 +18,15 @@
package org.apache.kyuubi.engine.spark
import java.io.File
import java.nio.file.{Files, Path, Paths}
import java.nio.file.{Files, Path, Paths, StandardOpenOption}
import java.time.Duration
import java.util.concurrent.{Executors, TimeUnit}
import org.scalatest.time.SpanSugar._
import org.apache.kyuubi.{KerberizedTestHelper, KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.ENGINE_LOG_TIMEOUT
import org.apache.kyuubi.service.ServiceUtils
class SparkProcessBuilderSuite extends KerberizedTestHelper {
@ -179,6 +181,29 @@ class SparkProcessBuilderSuite extends KerberizedTestHelper {
atomicTest()
}
}
test("overwrite log file should cleanup before write") {
val fakeWorkDir = Files.createTempDirectory("fake")
val conf = KyuubiConf()
conf.set(ENGINE_LOG_TIMEOUT, Duration.ofDays(1).toMillis)
val builder1 = new FakeSparkProcessBuilder(conf) {
override val workingDir: Path = fakeWorkDir
}
val file1 = builder1.engineLog
Files.write(file1.toPath, "a".getBytes(), StandardOpenOption.APPEND)
assert(file1.length() == 1)
Files.write(file1.toPath, "a".getBytes(), StandardOpenOption.APPEND)
assert(file1.length() == 2)
file1.setLastModified(System.currentTimeMillis() - Duration.ofDays(1).toMillis - 1000)
val builder2 = new FakeSparkProcessBuilder(conf) {
override val workingDir: Path = fakeWorkDir
}
val file2 = builder2.engineLog
assert(file1.getAbsolutePath == file2.getAbsolutePath)
Files.write(file2.toPath, "a".getBytes(), StandardOpenOption.APPEND)
assert(file2.length() == 1)
}
}
class FakeSparkProcessBuilder(config: KyuubiConf)